From 57350d1e49f2c0d2da0564977c92b7e20756797e Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 04:37:21 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20separate?= =?UTF-8?q?=20role=20tuning=20from=20shared=20Store=20configuration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/sink/config_command_test.go | 18 +- cmd/sink/config_path_test.go | 12 +- cmd/sink/dead_letter_command.go | 5 +- cmd/sink/dead_letter_command_test.go | 26 +- cmd/sink/lua_command.go | 26 +- cmd/sink/lua_command_test.go | 10 +- cmd/sink/main.go | 38 +- internal/app/gateway.go | 2 +- internal/app/kafka.go | 2 +- internal/app/logging_test.go | 4 +- internal/app/role_contract_test.go | 121 +++ internal/app/service.go | 31 +- internal/app/transport.go | 6 +- internal/config/bytes_test.go | 61 +- internal/config/config.go | 11 +- internal/config/config_test.go | 745 +++--------------- internal/config/file.go | 90 +-- internal/config/gateway.go | 16 +- internal/config/kafka.go | 48 +- internal/config/limits_test.go | 135 +--- internal/config/load.go | 145 ++-- internal/config/logging_test.go | 8 +- internal/config/memory_test.go | 8 +- internal/config/roles_test.go | 93 +-- internal/config/secrets_test.go | 12 +- internal/config/service.go | 79 +- internal/config/storage.go | 9 - internal/forwarding/budget.go | 7 +- internal/gateway/batch.go | 8 +- internal/gateway/gateway_test.go | 2 +- internal/gateway/methods.go | 25 +- internal/gateway/response_budget_test.go | 38 + internal/gateway/server.go | 43 +- internal/logging/logging_test.go | 2 +- internal/protocol/memory.go | 14 +- internal/service/admission.go | 14 +- internal/service/batcher.go | 31 +- internal/service/batching_regressions_test.go | 32 + internal/service/batching_server.go | 12 +- internal/service/caller_deadline_test.go | 45 ++ internal/service/native.go | 9 +- internal/service/options.go | 27 +- internal/service/query.go | 4 +- internal/service/read.go | 2 +- internal/service/server.go | 2 + internal/service/server_test.go | 3 +- internal/service/sync_capacity_test.go | 8 + internal/service/write.go | 12 +- 48 files changed, 903 insertions(+), 1198 deletions(-) create mode 100644 internal/app/role_contract_test.go create mode 100644 internal/gateway/response_budget_test.go create mode 100644 internal/service/caller_deadline_test.go diff --git a/cmd/sink/config_command_test.go b/cmd/sink/config_command_test.go index 519c337..08fe2c7 100644 --- a/cmd/sink/config_command_test.go +++ b/cmd/sink/config_command_test.go @@ -7,21 +7,9 @@ import ( ) func TestConfigCheckDoesNotOpenDependenciesOrPrintValues(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://127.0.0.1:1 - kafka: - enabled: true - brokers: [127.0.0.1:1] - topic: - name: mutations - consumer: - group_id: workers -`) - args := []string{"config", "check", "--config", path} + path := writeConfig(t, "mode: engine\n") + shared := writeConfig(t, "name: primary\nstorage:\n driver: mongodb\n mongodb: {uri: 'mongodb://127.0.0.1:1'}\nkafka:\n enabled: true\n brokers: ['127.0.0.1:1']\n topic: {name: mutations}\n") + args := []string{"config", "check", "--config", path, "--store-config", shared} var stdout, stderr bytes.Buffer if err := executeCommand(args, &stdout, &stderr); err != nil { t.Fatal(err) diff --git a/cmd/sink/config_path_test.go b/cmd/sink/config_path_test.go index cd4f9bf..92900a0 100644 --- a/cmd/sink/config_path_test.go +++ b/cmd/sink/config_path_test.go @@ -8,19 +8,19 @@ import ( func TestParseConfigPath(t *testing.T) { args := []string{"--config", "/etc/sink/config.yaml"} - path, err := parseConfigPath(args) + path, err := parseConfigPaths(args) if err != nil { - t.Fatalf("parseConfigPath() error = %v", err) + t.Fatalf("parseConfigPaths() error = %v", err) } - if path != "/etc/sink/config.yaml" { - t.Fatalf("parseConfigPath() = %q", path) + if path.Component != "/etc/sink/config.yaml" { + t.Fatalf("parseConfigPaths() = %q", path) } } func TestParseConfigPathRequiresFlag(t *testing.T) { - _, err := parseConfigPath(nil) + _, err := parseConfigPaths(nil) if err == nil || err.Error() != "--config is required" { - t.Fatalf("parseConfigPath() error = %v", err) + t.Fatalf("parseConfigPaths() error = %v", err) } } diff --git a/cmd/sink/dead_letter_command.go b/cmd/sink/dead_letter_command.go index 40ff4bb..57b20ad 100644 --- a/cmd/sink/dead_letter_command.go +++ b/cmd/sink/dead_letter_command.go @@ -28,11 +28,12 @@ type deadLetterReport struct { func runDeadLetterCommand(args []string, stdout io.Writer, stderr io.Writer) error { if len(args) == 0 || (args[0] != "inspect" && args[0] != "replay") { - return errors.New("usage: sink dlq inspect|replay --config FILE --store NAME --partition N --offset N --count N") + return errors.New("usage: sink dlq inspect|replay --config FILE --store-config FILE --store NAME --partition N --offset N --count N") } flags := flag.NewFlagSet("dlq "+args[0], flag.ContinueOnError) flags.SetOutput(stderr) configPath := flags.String("config", "", "server configuration") + storePath := flags.String("store-config", "", "shared Store configuration") store := flags.String("store", "", "configured store") partition := flags.Int("partition", -1, "exact dead-letter partition") offset := flags.Int64("offset", -1, "first dead-letter offset") @@ -43,7 +44,7 @@ func runDeadLetterCommand(args []string, stdout io.Writer, stderr io.Writer) err if flags.NArg() != 0 || *configPath == "" || *store == "" || *partition < 0 || *partition > 1<<31-1 || *offset < 0 || *count < 1 || *count > 100 { return errors.New("config, store, partition, offset, and a count between 1 and 100 are required") } - loaded, err := config.Load(*configPath) + loaded, err := config.Load(*configPath, *storePath) if err != nil { return err } diff --git a/cmd/sink/dead_letter_command_test.go b/cmd/sink/dead_letter_command_test.go index e1c8ab8..1cf91be 100644 --- a/cmd/sink/dead_letter_command_test.go +++ b/cmd/sink/dead_letter_command_test.go @@ -70,22 +70,22 @@ func TestDeadLetterReplayPreservesPublisherRouting(t *testing.T) { if driver == config.DriverMongoDB { backend = " mongodb:\n uri: mongodb://127.0.0.1:1\n" } - contents := fmt.Sprintf(`mode: engine + contents := fmt.Sprintf(`name: primary storage: - name: primary driver: %s -%s kafka: - enabled: true - brokers: [%q] - topic: - name: source - replication_factor: 1 - min_insync_replicas: 1 - dead_letter: - topic: source.dlq +%skafka: + enabled: true + brokers: [%q] + topic: + name: source + replication_factor: 1 + min_insync_replicas: 1 + dead_letter: + topic: source.dlq `, driver, backend, cluster.ListenAddrs()[0]) - configPath := writeConfig(t, contents) - args := []string{"replay", "--config", configPath, "--store", "primary", "--partition", strconv.Itoa(int(letter.Partition)), "--offset", strconv.FormatInt(letter.Offset, 10)} + sharedPath := writeConfig(t, contents) + configPath := writeConfig(t, "mode: engine\n") + args := []string{"replay", "--config", configPath, "--store-config", sharedPath, "--store", "primary", "--partition", strconv.Itoa(int(letter.Partition)), "--offset", strconv.FormatInt(letter.Offset, 10)} var stdout, stderr bytes.Buffer if err := runDeadLetterCommand(args, &stdout, &stderr); err != nil { t.Fatalf("replay: %v stdout=%s stderr=%s", err, &stdout, &stderr) diff --git a/cmd/sink/lua_command.go b/cmd/sink/lua_command.go index b07a4c3..9ca324c 100644 --- a/cmd/sink/lua_command.go +++ b/cmd/sink/lua_command.go @@ -23,14 +23,15 @@ import ( ) type luaTestFlags struct { - script string - config string - cases string - encoding string - current string - incoming string - expected string - observedAt string + script string + config string + storeConfig string + cases string + encoding string + current string + incoming string + expected string + observedAt string } type luaTestCaseFile struct { @@ -84,7 +85,7 @@ func runLuaTestCommand(args []string, stdout io.Writer, stderr io.Writer) error } luaOptions := merge.LuaOptions{} if parsed.config != "" { - loaded, loadErr := config.Load(parsed.config) + loaded, loadErr := config.Load(parsed.config, parsed.storeConfig) if loadErr != nil { return fmt.Errorf("load Lua test limits from config: %w", loadErr) } @@ -117,7 +118,8 @@ func parseLuaTestFlags(args []string, stderr io.Writer) (luaTestFlags, error) { flags := flag.NewFlagSet("sink lua test", flag.ContinueOnError) flags.SetOutput(stderr) flags.StringVar(&parsed.script, "script", "", "path to the Lua merge script") - flags.StringVar(&parsed.config, "config", "", "optional Sink config whose service.merge.lua limits are applied") + flags.StringVar(&parsed.storeConfig, "store-config", "", "Store configuration used with --config") + flags.StringVar(&parsed.config, "config", "", "optional Sink config whose execution.merge.lua limits are applied") flags.StringVar(&parsed.cases, "cases", "", "path to one YAML case or a directory of YAML cases") flags.StringVar(&parsed.encoding, "encoding", "", "single-case document encoding: json or bson") flags.StringVar(&parsed.current, "current", "", "optional single-case current JSON or Extended JSON document") @@ -132,6 +134,10 @@ func parseLuaTestFlags(args []string, stderr io.Writer) (luaTestFlags, error) { } parsed.script = strings.TrimSpace(parsed.script) parsed.config = strings.TrimSpace(parsed.config) + parsed.storeConfig = strings.TrimSpace(parsed.storeConfig) + if parsed.storeConfig != "" && parsed.config == "" { + return parsed, errors.New("--store-config requires --config") + } parsed.cases = strings.TrimSpace(parsed.cases) parsed.encoding = strings.TrimSpace(parsed.encoding) parsed.current = strings.TrimSpace(parsed.current) diff --git a/cmd/sink/lua_command_test.go b/cmd/sink/lua_command_test.go index 11a084f..a4a86d1 100644 --- a/cmd/sink/lua_command_test.go +++ b/cmd/sink/lua_command_test.go @@ -234,13 +234,9 @@ end`) func TestLuaTestCommandUsesConfiguredLimitsWithoutOpeningBackends(t *testing.T) { fixture := newLuaCommandFixture(t, `return function(current, incoming) return incoming end`) + shared := fixture.write(t, "store.yaml", "name: primary\nstorage:\n driver: mongodb\n mongodb: {uri: mongodb://unreachable.invalid:27017}\n") config := fixture.write(t, "sink.yaml", `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://unreachable.invalid:27017 -service: +execution: merge: lua: max_source_bytes: 1 @@ -248,7 +244,7 @@ service: incoming := fixture.write(t, "incoming.json", `{}`) args := []string{ "--script", fixture.script, - "--config", config, + "--config", config, "--store-config", shared, "--encoding", "json", "--incoming", incoming, "--observed-at", "2026-08-31T10:20:30Z", diff --git a/cmd/sink/main.go b/cmd/sink/main.go index 43fea56..b4793d4 100644 --- a/cmd/sink/main.go +++ b/cmd/sink/main.go @@ -48,22 +48,22 @@ func executeCommand(args []string, stdout io.Writer, stderr io.Writer) error { return runConfigCommand(args[1:], stdout) } } - configPath, err := parseConfigPath(args) + paths, err := parseConfigPaths(args) if err != nil { return err } - return run(configPath) + return run(paths) } func runConfigCommand(args []string, stdout io.Writer) error { if len(args) == 0 || args[0] != "check" { - return errors.New("usage: sink config check --config FILE") + return errors.New("usage: sink config check --config FILE [--store-config FILE]") } - configPath, err := parseConfigPath(args[1:]) + paths, err := parseConfigPaths(args[1:]) if err != nil { return err } - loaded, err := config.Load(configPath) + loaded, err := config.Load(paths.Component, paths.Store) if err != nil { return err } @@ -76,27 +76,35 @@ func runConfigCommand(args []string, stdout io.Writer) error { return err } -func parseConfigPath(args []string) (string, error) { +type configPaths struct { + Component string + Store string +} + +func parseConfigPaths(args []string) (configPaths, error) { + var paths configPaths flags := flag.NewFlagSet("sink", flag.ContinueOnError) flags.SetOutput(io.Discard) - configPath := flags.String("config", "", "path to the YAML configuration file") + flags.StringVar(&paths.Component, "config", "", "path to the component YAML configuration") + flags.StringVar(&paths.Store, "store-config", "", "path to the shared Store YAML (required for Engine/Worker)") if err := flags.Parse(args); err != nil { - return "", fmt.Errorf("parse command arguments: %w", err) + return paths, fmt.Errorf("parse command arguments: %w", err) } if flags.NArg() != 0 { - return "", fmt.Errorf("unexpected command argument %q", flags.Arg(0)) + return paths, fmt.Errorf("unexpected command argument %q", flags.Arg(0)) } - trimmed := strings.TrimSpace(*configPath) - if trimmed == "" { - return "", errors.New("--config is required") + paths.Component = strings.TrimSpace(paths.Component) + paths.Store = strings.TrimSpace(paths.Store) + if paths.Component == "" { + return paths, errors.New("--config is required") } - return trimmed, nil + return paths, nil } type reportedError struct{ error } -func run(configPath string) (runErr error) { - loaded, err := config.Load(configPath) +func run(paths configPaths) (runErr error) { + loaded, err := config.Load(paths.Component, paths.Store) if err != nil { return err } diff --git a/internal/app/gateway.go b/internal/app/gateway.go index b59821c..8d2cbf2 100644 --- a/internal/app/gateway.go +++ b/internal/app/gateway.go @@ -10,7 +10,7 @@ func newGateway(opts Options) (*Application, error) { if err != nil { return nil, err } - gatewayOpts := gateway.Options{Memory: memory, Gateway: loaded.Gateway, Request: loaded.Service.Request, MaxMessageBytes: max(loaded.GRPC.MaxSendMessageBytes, loaded.GRPC.MaxReceiveMessageBytes)} + gatewayOpts := gateway.Options{Memory: memory, Gateway: loaded.Gateway, Request: loaded.Service.Request, MaxResponseBytes: loaded.GRPC.MaxSendMessageBytes, MaxMessageBytes: max(loaded.GRPC.MaxSendMessageBytes, loaded.GRPC.MaxReceiveMessageBytes)} server, err := gateway.New(gatewayOpts) if err != nil { return nil, err diff --git a/internal/app/kafka.go b/internal/app/kafka.go index 266d07d..40412f0 100644 --- a/internal/app/kafka.go +++ b/internal/app/kafka.go @@ -62,7 +62,7 @@ func (app *Application) configureWorker(server *service.Server, observed *sinkme Brokers: configured.Kafka.Brokers, Store: configured.Name, Topic: configured.Kafka.Topic.Name, GroupID: configured.Kafka.Consumer.GroupID, DeadLetterTopic: configured.Kafka.DeadLetter.Topic, Handler: processor, - MaxPollRecords: min(configured.Kafka.Consumer.MaxPollRecords, loaded.Service.Request.MaxOperations), + MaxPollRecords: configured.Kafka.Consumer.MaxPollRecords, MaxRetryAttempts: configured.Kafka.Consumer.Retry.MaxAttempts, RetryBackoff: configured.Kafka.Consumer.Retry.Backoff, MaxRetryBackoff: configured.Kafka.Consumer.Retry.MaxBackoff, Metrics: observed, diff --git a/internal/app/logging_test.go b/internal/app/logging_test.go index e2a3c17..d1bc0d1 100644 --- a/internal/app/logging_test.go +++ b/internal/app/logging_test.go @@ -19,7 +19,7 @@ import ( ) func TestRPCDiagnosticsCaptureApplicationFailuresWithoutPayload(t *testing.T) { - loaded, err := config.Decode(strings.NewReader("mode: engine\nstorage:\n name: primary\n driver: mongodb\n mongodb:\n uri: mongodb://localhost:27017\n")) + loaded, err := config.Decode(strings.NewReader("mode: engine\n"), strings.NewReader("name: primary\nstorage:\n driver: mongodb\n mongodb:\n uri: mongodb://localhost:27017\n")) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestRPCDiagnosticsCaptureApplicationFailuresWithoutPayload(t *testing.T) { } func TestStreamDiagnosticsPreserveOutcomeAndExcludeHealth(t *testing.T) { - loaded, err := config.Decode(strings.NewReader("mode: gateway\ngateway:\n routes: [{store: primary, target: '127.0.0.1:8080', tls: {insecure: true}}]\nlogging:\n level: debug\n")) + loaded, err := config.Decode(strings.NewReader("mode: gateway\nforwarding:\n routes: [{store: primary, target: '127.0.0.1:8080', tls: {insecure: true}}]\nlogging:\n level: debug\n"), nil) if err != nil { t.Fatal(err) } diff --git a/internal/app/role_contract_test.go b/internal/app/role_contract_test.go new file mode 100644 index 0000000..f2e77c7 --- /dev/null +++ b/internal/app/role_contract_test.go @@ -0,0 +1,121 @@ +package app + +import ( + "context" + "strings" + "testing" + "time" + + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/config" + "github.com/liran/sink/internal/storage" + "github.com/liran/sink/internal/storage/memory" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type deadlineReadStorage struct { + storage.Storage + seen chan context.Context + release chan struct{} +} + +func (s *deadlineReadStorage) Read(ctx context.Context, req storage.ReadRequest) (storage.ReadResponse, error) { + s.seen <- ctx + select { + case <-s.release: + return s.Storage.Read(ctx, req) + case <-ctx.Done(): + var empty storage.ReadResponse + return empty, ctx.Err() + } +} + +func TestEngineOnlyAcceptsForwardingAndHonorsGatewayContext(t *testing.T) { + shared := "name: primary\nstorage: {driver: mongodb, mongodb: {uri: 'mongodb://127.0.0.1:1'}}" + loaded, err := config.Decode(strings.NewReader("mode: engine\ngrpc: {address: '127.0.0.1:0'}"), strings.NewReader(shared)) + if err != nil { + t.Fatal(err) + } + pool, err := newMemory(loaded) + if err != nil { + t.Fatal(err) + } + backend := &deadlineReadStorage{Storage: memory.New(), seen: make(chan context.Context, 2), release: make(chan struct{})} + engine := &Application{config: loaded, memory: pool, storage: backend} + core, err := engine.newService(nil) + if err != nil { + t.Fatal(err) + } + if err := engine.configureServer(core, nil); err != nil { + t.Fatal(err) + } + t.Cleanup(engine.Close) + go engine.grpcServer.Serve(engine.listener) + engineConn, err := grpc.NewClient(engine.listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + defer engineConn.Close() + direct := sink.NewSinkClient(engineConn) + empty := &sink.ReadRequest{} + if _, err := direct.Read(t.Context(), empty); status.Code(err) != codes.Unimplemented { + t.Fatalf("Engine exposes public RPC: %v", err) + } + component := "mode: gateway\ngrpc: {address: '127.0.0.1:0'}\nhealth: {address: '127.0.0.1:0'}\nrequest: {max_operations: 2000}\nforwarding:\n routes: [{store: primary, target: '" + engine.listener.Addr().String() + "', tls: {insecure: true}}]" + gatewayConfig, err := config.Decode(strings.NewReader(component), nil) + if err != nil { + t.Fatal(err) + } + opts := Options{Config: gatewayConfig} + gateway, err := New(t.Context(), opts) + if err != nil { + t.Fatal(err) + } + t.Cleanup(gateway.Close) + go gateway.grpcServer.Serve(gateway.listener) + conn, err := grpc.NewClient(gateway.listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + client := sink.NewSinkClient(conn) + addr := &sink.RecordAddress{Uri: "sink://primary/db/items/s:key"} + operation := &sink.ReadOperation{Address: addr} + request := &sink.ReadRequest{} + for range 1500 { + request.Operations = append(request.Operations, operation) + } + caller, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { _, err := client.Read(caller, request); done <- err }() + var observed context.Context + select { + case observed = <-backend.seen: + case <-time.After(5 * time.Second): + t.Fatal("Gateway request did not reach Engine") + } + if deadline, ok := observed.Deadline(); ok { + t.Fatalf("hidden execution deadline: %v", deadline) + } + cancel() + select { + case <-observed.Done(): + case <-time.After(time.Second): + t.Fatal("caller cancellation did not reach storage") + } + if err := <-done; status.Code(err) != codes.Canceled { + t.Fatalf("cancellation result: %v", err) + } + close(backend.release) + // More than 1000 operations in one Store is valid when Gateway permits it. + ctx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + response, err := client.Read(ctx, request) + if err != nil || len(response.GetResults()) != 1500 { + t.Fatalf("Engine reintroduced an operation cap: count=%d err=%v", len(response.GetResults()), err) + } +} diff --git a/internal/app/service.go b/internal/app/service.go index c340d32..1bd0d89 100644 --- a/internal/app/service.go +++ b/internal/app/service.go @@ -23,25 +23,16 @@ func (app *Application) newService(observed *sinkmetrics.Metrics) (*service.Serv return nil, err } serverOptions := service.Options{ - Memory: app.memory, - RequestTimeout: loaded.Service.Request.Timeout, - MaxInFlightRequests: loaded.Service.Execution.MaxRequests, - MaxInFlightBytes: loaded.Service.Execution.MaxBytes, - MaxAdmissionRequests: loaded.Service.Execution.Queue.MaxRequests, - MaxAdmissionBytes: loaded.Service.Execution.Queue.MaxBytes, - AdmissionWait: loaded.Service.Execution.Queue.MaxWait, - MaxPublishRequests: loaded.Service.Publish.MaxRequests, - MaxPublishBytes: loaded.Service.Publish.MaxBytes, - MaxScanRequests: loaded.Service.Execution.Scan.MaxRequests, - MaxScanBytes: loaded.Service.Execution.Scan.MaxBytes, - ScanAdmissionWait: loaded.Service.Execution.Scan.AdmissionWait, - MaxReadBytes: loaded.Service.Request.MaxReadBytes, - Storage: app.storage, - Lua: luaEngine, - Publisher: app.publisher, - MaxOperations: loaded.Service.Request.MaxOperations, - MaxMergeAttempts: loaded.Service.Merge.MaxAttempts, - Metrics: observed, + Memory: app.memory, + MaxReadBytes: max(loaded.GRPC.MaxSendMessageBytes, loaded.Service.Execution.MaxOutputBytes), + MaxSnapshotBytes: loaded.Service.Execution.MaxSnapshotBytes, + MaxOutputBytes: loaded.Service.Execution.MaxOutputBytes, + Storage: app.storage, + Lua: luaEngine, + Publisher: app.publisher, + MaxOperations: 0, + MaxMergeAttempts: loaded.Service.Merge.MaxAttempts, + Metrics: observed, } serverOptions.BoundStore = loaded.Storage.Name return service.New(serverOptions) @@ -66,7 +57,7 @@ func (app *Application) configureServer(sinkServer *service.Server, observed *si if err := app.configureGRPC(app.batchingServer, observed); err != nil { return err } - opts := engine.Options{MaxRequestBytes: loaded.GRPC.MaxReceiveMessageBytes, Metrics: observed, Service: app.batchingServer, Store: loaded.Storage.Name, MaxReadBytes: loaded.Service.Request.MaxReadBytes} + opts := engine.Options{MaxRequestBytes: loaded.GRPC.MaxReceiveMessageBytes, Metrics: observed, Service: app.batchingServer, Store: loaded.Storage.Name, MaxReadBytes: loaded.GRPC.MaxSendMessageBytes} forwardingServer, err := engine.New(opts) if err != nil { return err diff --git a/internal/app/transport.go b/internal/app/transport.go index 704eda8..cbad359 100644 --- a/internal/app/transport.go +++ b/internal/app/transport.go @@ -26,7 +26,7 @@ func (app *Application) configureGRPC(server sink.SinkServer, observed *sinkmetr return fmt.Errorf("listen for gRPC: %w", err) } serverOptions := make([]grpc.ServerOption, 0, 4) - memoryStats := &protocol.MemoryStats{Pool: app.memory, Timeout: app.config.Service.Request.Timeout} + memoryStats := &protocol.MemoryStats{Pool: app.memory} if app.memory != nil { serverOptions = append(serverOptions, grpc.StatsHandler(memoryStats)) } @@ -69,7 +69,9 @@ func (app *Application) configureGRPC(server sink.SinkServer, observed *sinkmetr serverOptions = append(serverOptions, grpc.ChainUnaryInterceptor(interceptors...)) serverOptions = append(serverOptions, grpc.ChainStreamInterceptor(streamInterceptors...)) grpcServer := grpc.NewServer(serverOptions...) - sink.RegisterSinkServer(grpcServer, server) + if app.config.Mode == config.ModeGateway { + sink.RegisterSinkServer(grpcServer, server) + } healthServer := health.NewServer() healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) for _, configured := range app.healthChecks { diff --git a/internal/config/bytes_test.go b/internal/config/bytes_test.go index 6900a57..43c9219 100644 --- a/internal/config/bytes_test.go +++ b/internal/config/bytes_test.go @@ -51,7 +51,7 @@ func TestByteSizeRejectsAmbiguousLossyAndOverflowValues(t *testing.T) { "9223372036854775808B", "8388608TiB", "9223372036854775808", } { t.Run(input, func(t *testing.T) { - _, err := Decode(strings.NewReader(minimalStorage + "grpc:\n max_receive_message_bytes: " + input + "\n")) + _, err := Decode(strings.NewReader("mode: engine\ngrpc:\n max_receive_message_bytes: "+input+"\n"), strings.NewReader(minimalStorage)) if err == nil || !strings.Contains(err.Error(), "byte size") { t.Fatalf("expected actionable byte size error, got %v", err) } @@ -60,59 +60,26 @@ func TestByteSizeRejectsAmbiguousLossyAndOverflowValues(t *testing.T) { } func TestAllByteLimitsAcceptHumanReadableUnits(t *testing.T) { - input := minimalStorage + ` kafka: - topic: - max_record_bytes: 900KiB - producer: - max_buffered_bytes: 64MiB -grpc: - max_receive_message_bytes: 64MiB - max_send_message_bytes: 64MiB -service: - request: - max_read_bytes: 16MiB - execution: - max_bytes: 256MiB - scan: - max_bytes: 128MiB - publish: - max_bytes: 256MiB - batching: - max_bytes: 16MiB - queue: - max_bytes: 128MiB - merge: - lua: - max_source_bytes: 64KiB - max_result_bytes: 16MiB + input := `mode: engine +memory: {max_bytes: 256MiB} +producer: {max_buffered_bytes: 64MiB} +batching: {max_bytes: 16MiB, queue: {max_bytes: 128MiB}} +execution: + max_snapshot_bytes: 16MiB + max_output_bytes: 64MiB + merge: + lua: {max_source_bytes: 64KiB, max_result_bytes: 16MiB} ` - human, err := Decode(strings.NewReader(input)) + human, err := Decode(strings.NewReader(input), strings.NewReader(kafkaStore)) if err != nil { t.Fatal(err) } - numeric := strings.NewReplacer("64MiB", "67108864", "900KiB", "921600", "16MiB", "16777216", "256MiB", "268435456", "128MiB", "134217728", "64KiB", "65536").Replace(input) - bytes, err := Decode(strings.NewReader(numeric)) + numeric := strings.NewReplacer("64MiB", "67108864", "16MiB", "16777216", "256MiB", "268435456", "128MiB", "134217728", "64KiB", "65536").Replace(input) + bytes, err := Decode(strings.NewReader(numeric), strings.NewReader(kafkaStore)) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(human, bytes) { - t.Fatal("human-readable limits changed resolved configuration values") - } -} - -func TestHumanReadableSizesStillEnforceResourceLimits(t *testing.T) { - for _, section := range []string{ - "request:\n max_read_bytes: 33MiB", - "execution:\n max_bytes: 17GiB", - "execution:\n scan:\n max_bytes: 257MiB", - "publish:\n max_bytes: 17GiB", - "batching:\n queue:\n max_bytes: 63MiB", - "merge:\n lua:\n max_source_bytes: 0B", - "request:\n max_operations: 1KiB", - } { - _, err := Decode(strings.NewReader(minimalStorage + "service:\n " + section + "\n")) - if err == nil { - t.Fatalf("invalid resource limit accepted: %s", section) - } + t.Fatal("byte units changed configuration") } } diff --git a/internal/config/config.go b/internal/config/config.go index dd3ae12..ee0d17a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -71,10 +71,13 @@ type Request struct { } type Execution struct { - MaxRequests int - MaxBytes int - Queue AdmissionQueue - Scan Scan + MaxSnapshotBytes int + MaxOutputBytes int + MongoDB MongoDB + MaxRequests int + MaxBytes int + Queue AdmissionQueue + Scan Scan } type AdmissionQueue struct { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c8b4a6a..64aae06 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,662 +1,151 @@ package config import ( + "io" "os" "path/filepath" + "reflect" "strings" "testing" "time" ) -func TestLoadConfigEngineDefaults(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -`) - loaded, err := Load(path) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - if loaded.Mode != ModeEngine || loaded.GRPC.Address != ":8080" { - t.Fatalf("Load() = %#v", loaded) - } - if loaded.Prometheus.Enabled || loaded.Prometheus.Address != ":9090" || loaded.Health.Address != ":8081" { - t.Fatalf("Load() health/Prometheus = %#v / %#v", loaded.Health, loaded.Prometheus) - } - configured := loaded.Storage - if configured.Name != "primary" || configured.Driver != DriverMongoDB || configured.MongoDB.URI != "mongodb://mongodb:27017" { - t.Fatalf("Load() storage = %#v", configured) - } - if loaded.Service.Request.MaxOperations != 1000 || loaded.Service.Merge.MaxAttempts != 3 || loaded.ShutdownTimeout != 15*time.Second { - t.Fatalf("Load() service defaults = %#v", loaded) - } - if loaded.Service.Batching.MaxWait != 2*time.Millisecond || - loaded.Service.Batching.MaxOperations != 1000 || loaded.Service.Batching.MaxBytes != 16<<20 || - loaded.Service.Batching.Queue.MaxOperations != 10_000 || loaded.Service.Batching.Queue.MaxBytes != 128<<20 { - t.Fatalf("Load() batching defaults = %#v", loaded) - } - if loaded.Service.Merge.Lua.Timeout != 100*time.Millisecond || loaded.Service.Merge.Lua.MaxSourceBytes != 64<<10 || - loaded.Service.Merge.Lua.MaxResultBytes != 16<<20 || loaded.Service.Merge.Lua.MaxCachedPrograms != 256 || - loaded.Service.Merge.Lua.MaxInstructions != 1_000_000 { - t.Fatalf("Load() Lua defaults = %#v", loaded.Service.Merge.Lua) - } - if loaded.GRPC.MaxReceiveMessageBytes != 64<<20 || loaded.GRPC.MaxSendMessageBytes != 64<<20 { - t.Fatalf("Load() gRPC limits = %#v", loaded) - } - if configured.Kafka.Enabled { - t.Fatalf("Load() Kafka configuration = %#v", configured.Kafka) - } -} - -func TestLoadConfigDisablesKafkaByDefault(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - brokers: [kafka:9092] - topic: - name: sink-mutations -`) - loaded, err := Load(path) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - configured := loaded.Storage.Kafka - if configured.Enabled { - t.Fatalf("Load() Kafka = %#v", configured) - } -} +const minimalStorage = "name: primary\nstorage:\n driver: mongodb\n mongodb:\n uri: mongodb://127.0.0.1:1\n" +const gatewayConfig = "mode: gateway\nforwarding:\n routes: [{store: primary, target: '127.0.0.1:1', tls: {insecure: true}}]\n" +const kafkaStore = minimalStorage + "kafka:\n enabled: true\n brokers: [127.0.0.1:1]\n topic: {name: mutations}\n" -func TestLoadConfigBatchingSettings(t *testing.T) { - path := writeConfig(t, `mode: engine -grpc: - max_receive_message_bytes: 1048576 -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -service: - request: - max_operations: 2000 - batching: - max_wait: 5ms - max_operations: 500 - max_bytes: 524288 - queue: - max_operations: 2500 - max_bytes: 2097152 -`) - loaded, err := Load(path) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - if loaded.Service.Batching.MaxWait != 5*time.Millisecond || - loaded.Service.Batching.MaxOperations != 500 || loaded.Service.Batching.MaxBytes != 524288 || - loaded.Service.Batching.Queue.MaxOperations != 2500 || loaded.Service.Batching.Queue.MaxBytes != 2097152 { - t.Fatalf("Load() batching settings = %#v", loaded) - } -} - -func TestLoadConfigRejectsBatchingSwitch(t *testing.T) { - for _, enabled := range []string{"true", "false"} { - t.Run(enabled, func(t *testing.T) { - contents := `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -service: - batching: - enabled: ` + enabled + "\n" - path := writeConfig(t, contents) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "field enabled not found in type config.batchingFile") { - t.Fatalf("Load() accepted removed batching switch: %v", err) - } - }) - } -} - -func TestLoadConfigRejectsUnsafeBatchingLimits(t *testing.T) { - tests := []struct { - name string - batching string - wantError string - }{ - { - name: "batch exceeds service operation limit", - batching: "max_operations: 1001", - wantError: "service.batching.max_operations cannot exceed service.request.max_operations", - }, - { - name: "queue cannot hold one request", - batching: "queue:\n max_operations: 999", - wantError: "service.batching.queue.max_operations must cover one server request and one batch", - }, - { - name: "byte queue cannot hold one gRPC message", - batching: "queue:\n max_bytes: 1048576", - wantError: "service.batching.queue.max_bytes must cover one gRPC request and one batch", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - contents := `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -service: - batching: - ` + test.batching + "\n" - path := writeConfig(t, contents) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), test.wantError) { - t.Fatalf("Load() error = %v", err) - } - }) - } -} - -func TestLoadConfigRejectsMultipleStorages(t *testing.T) { - path := writeConfig(t, ` -health: - address: ":9090" -storages: - - name: mongo-main - driver: mongodb - mongodb: - uri: mongodb://mongo-main:27017 - metadata_field: __revision - - name: mongo-archive - driver: mongodb - mongodb: - uri: mongodb://mongo-archive:27017 - - name: search-main - driver: elasticsearch - search: - endpoints: - - http://search-1:9200 - - http://search-2:9200 - api_key: test-api-key -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "field storages not found") { - t.Fatalf("plural storage configuration was accepted: %v", err) - } -} - -func TestLoadConfigOpenSearchBasicAuthentication(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: search-main - driver: opensearch - search: - endpoints: - - https://search:9200 - username: sink - password: test-password -`) - loaded, err := Load(path) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - configured := loaded.Storage - if configured.Driver != DriverOpenSearch || configured.Search.Username != "sink" || configured.Search.Password != "test-password" { - t.Fatalf("Load() storage = %#v", configured) - } -} - -func TestLoadConfigRejectsConflictingSearchAuthentication(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: search-main - driver: elasticsearch - search: - endpoints: [http://search:9200] - username: sink - password: test-password - api_key: test-api-key -`) - _, err := Load(path) - if err == nil { - t.Fatal("Load() error = nil") +func writeConfig(t *testing.T, contents string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(contents), 0600); err != nil { + t.Fatal(err) } + return path } -func TestLoadConfigWorkerSettings(t *testing.T) { - path := writeConfig(t, ` -mode: worker -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - enabled: true - brokers: - - kafka-1:9092 - - kafka-2:9092 - topic: - name: sink-mutations - partitions: 12 - replication_factor: 3 - retention: 48h - consumer: - group_id: sink-workers - max_poll_records: 250 - retry: - max_attempts: 4 - backoff: 20ms - max_backoff: 200ms - dead_letter: - topic: sink-dead-letters -service: - request: - max_operations: 2000 - merge: - max_attempts: 5 - lua: - timeout: 250ms - max_source_bytes: 32768 - max_result_bytes: 1048576 - max_cached_programs: 128 - max_instructions: 2000000 -shutdown_timeout: 30s -`) - loaded, err := Load(path) +func TestSharedStoreWithIndependentRoleSettings(t *testing.T) { + storePath := writeConfig(t, kafkaStore) + enginePath := writeConfig(t, "mode: engine\nproducer: {max_buffered_bytes: 8MiB}\nexecution:\n mongodb: {max_concurrent_writes: 17}\n") + workerPath := writeConfig(t, "mode: worker\nconsumer: {group_id: workers, max_poll_records: 2000}\nexecution:\n mongodb: {max_concurrent_writes: 7}\n") + engine, err := Load(enginePath, storePath) if err != nil { - t.Fatalf("Load() error = %v", err) - } - configured := loaded.Storage - if loaded.Mode != ModeWorker || len(configured.Kafka.Brokers) != 2 || configured.Kafka.Consumer.MaxPollRecords != 250 { - t.Fatalf("Load() = %#v", loaded) - } - if configured.Kafka.DeadLetter.Topic != "sink-dead-letters" || configured.Kafka.Consumer.Retry.MaxAttempts != 4 || configured.Kafka.Consumer.Retry.Backoff != 20*time.Millisecond || configured.Kafka.Consumer.Retry.MaxBackoff != 200*time.Millisecond { - t.Fatalf("Load() Kafka retry settings = %#v", configured.Kafka) - } - if configured.Kafka.Topic.Partitions != 12 || - configured.Kafka.Topic.ReplicationFactor != 3 || configured.Kafka.Topic.Retention != 48*time.Hour { - t.Fatalf("Load() Kafka topic settings = %#v", configured.Kafka) - } - if loaded.Service.Request.MaxOperations != 2000 || loaded.Service.Merge.MaxAttempts != 5 || loaded.ShutdownTimeout != 30*time.Second { - t.Fatalf("Load() service settings = %#v", loaded) + t.Fatal(err) } - if loaded.Service.Merge.Lua.Timeout != 250*time.Millisecond || loaded.Service.Merge.Lua.MaxSourceBytes != 32768 || - loaded.Service.Merge.Lua.MaxResultBytes != 1048576 || loaded.Service.Merge.Lua.MaxCachedPrograms != 128 || - loaded.Service.Merge.Lua.MaxInstructions != 2_000_000 { - t.Fatalf("Load() Lua settings = %#v", loaded.Service.Merge.Lua) - } -} - -func TestLoadConfigEngineAllowsKafkaWithoutConsumerGroup(t *testing.T) { - path := writeConfig(t, ` -mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - enabled: true - brokers: [kafka:9092] - topic: - name: sink-mutations -`) - loaded, err := Load(path) + worker, err := Load(workerPath, storePath) if err != nil { - t.Fatalf("Load() error = %v", err) - } - configured := loaded.Storage.Kafka - if configured.Consumer.GroupID != "" || configured.DeadLetter.Topic != "sink-mutations.dlq" { - t.Fatalf("Load() Kafka = %#v", configured) - } -} - -func TestLoadConfigRejectsNonPositiveLuaLimits(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -service: - merge: - lua: - max_source_bytes: 0 -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "service.merge.lua.max_source_bytes") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigRejectsDuplicateStorageNames(t *testing.T) { - path := writeConfig(t, ` -storages: - - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongo-1:27017 - - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongo-2:27017 -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "field storages not found") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigRequiresAtLeastOneStorage(t *testing.T) { - path := writeConfig(t, ` -mode: engine - -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "require singular storage") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigRequiresStorageNameAndDriver(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: "" - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "storage.name") { - t.Fatalf("Load() error = %v", err) - } - - path = writeConfig(t, `mode: engine -storage: - name: primary - mongodb: - uri: mongodb://mongodb:27017 -`) - _, err = Load(path) - if err == nil || !strings.Contains(err.Error(), "storage.driver must be") { - t.Fatalf("Load() error = %v", err) + t.Fatal(err) } -} - -func TestLoadConfigRejectsBindings(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - bindings: [] -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "field bindings not found") { - t.Fatalf("Load() error = %v", err) + if engine.Storage.Name != worker.Storage.Name || engine.Storage.MongoDB.URI != worker.Storage.MongoDB.URI || !reflect.DeepEqual(engine.Storage.Kafka.Topic, worker.Storage.Kafka.Topic) { + t.Fatal("shared identity or dependencies diverged") } -} - -func TestLoadConfigRejectsPartialKafkaConfiguration(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - enabled: true - brokers: [kafka:9092] -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "storage.kafka") { - t.Fatalf("Load() error = %v", err) + if engine.Storage.MongoDB.MaxConcurrentWrites != 17 || worker.Storage.MongoDB.MaxConcurrentWrites != 7 || worker.Storage.Kafka.Consumer.MaxPollRecords != 2000 || engine.Storage.Kafka.Producer.MaxBufferedBytes != 8<<20 { + t.Fatal("role tuning was not independent") } -} - -func TestLoadConfigWorkerRequiresGroupForEveryKafkaStore(t *testing.T) { - path := writeConfig(t, ` -mode: worker -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - enabled: true - brokers: [kafka:9092] - topic: - name: sink-mutations -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "storage.kafka.consumer.group_id") { - t.Fatalf("Load() error = %v", err) + if worker.Storage.Kafka.Consumer.GroupID != "workers" || engine.Storage.Kafka.Consumer.GroupID != "" { + t.Fatal("consumer leaked between roles") } } -func TestLoadConfigRejectsTopLevelKafkaConfiguration(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -kafka: - brokers: [kafka:9092] - topic: - name: sink-mutations -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "field kafka not found") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigWorkerRequiresAtLeastOneKafkaStore(t *testing.T) { - path := writeConfig(t, ` -mode: worker -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "worker requires storage.kafka.enabled") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigRejectsUnknownFields(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -unexpected: true -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "field unexpected not found") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigRejectsNonPositiveValues(t *testing.T) { - path := writeConfig(t, `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 -service: - request: - max_operations: 0 -`) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), "service.request.max_operations must be a positive integer") { - t.Fatalf("Load() error = %v", err) - } -} - -func TestLoadConfigDefaultsWorkerDeadLetterTopic(t *testing.T) { - path := writeConfig(t, ` -mode: worker -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - enabled: true - brokers: [kafka:9092] - topic: - name: sink-mutations - consumer: - group_id: sink-workers -`) - loaded, err := Load(path) +func TestComponentDefaults(t *testing.T) { + engine, err := Decode(strings.NewReader("mode: engine\n"), strings.NewReader(minimalStorage)) if err != nil { - t.Fatalf("Load() error = %v", err) + t.Fatal(err) } - configured := loaded.Storage.Kafka - if configured.DeadLetter.Topic != "sink-mutations.dlq" || configured.Consumer.MaxPollRecords != 500 || - configured.Consumer.Retry.MaxAttempts != 10 || configured.Consumer.Retry.Backoff != 100*time.Millisecond || - configured.Consumer.Retry.MaxBackoff != 10*time.Second || - configured.Topic.Partitions != 4 || configured.Topic.ReplicationFactor != 2 || - configured.Topic.Retention != 72*time.Hour { - t.Fatalf("Kafka defaults = %#v", configured) + if engine.Service.Request.Timeout != 0 || engine.Service.Request.MaxOperations != 0 || engine.Service.Request.MaxReadBytes != 0 { + t.Fatal("Engine acquired public request limits") } -} - -func TestLoadConfigRejectsInvalidKafkaTopicSettings(t *testing.T) { - tests := []struct { - name string - setting string - wantError string - }{ - { - name: "zero partitions", - setting: "partitions: 0", - wantError: "storage.kafka.topic.partitions must be a positive integer", - }, - { - name: "zero replication factor", - setting: "replication_factor: 0", - wantError: "storage.kafka.topic.replication_factor must be a positive integer", - }, - { - name: "zero retention", - setting: "retention: 0s", - wantError: "storage.kafka.topic.retention must be a positive duration", - }, + if engine.Service.Batching.MaxOperations != 1000 || engine.Service.Batching.MaxWait != 2*time.Millisecond || engine.Service.Batching.Queue.MaxBytes != 128<<20 { + t.Fatal("batch defaults changed") } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - contents := `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongodb:27017 - kafka: - enabled: true - brokers: [kafka:9092] - topic: - name: sink-mutations - ` + test.setting + "\n" - path := writeConfig(t, contents) - _, err := Load(path) - if err == nil || !strings.Contains(err.Error(), test.wantError) { - t.Fatalf("Load() error = %v", err) - } - }) + if engine.Service.Execution.MaxSnapshotBytes != 32<<20 || engine.Service.Execution.MaxOutputBytes != 32<<20 || engine.Service.Merge.Lua.MaxResultBytes != 16<<20 { + t.Fatal("execution defaults changed") } -} - -func TestLoadConfigDoesNotReadLegacyEnvironmentVariables(t *testing.T) { - t.Setenv("SINK_MODE", "worker") - t.Setenv("SINK_MONGODB_URI", "mongodb://legacy-environment:27017") - path := writeConfig(t, ` -mode: engine -storage: - name: configured - driver: mongodb - mongodb: - uri: mongodb://configured:27017 -`) - loaded, err := Load(path) - if err != nil { - t.Fatalf("Load() error = %v", err) + if engine.Storage.Kafka.Enabled || engine.Prometheus.Enabled || engine.Health.Address != ":8081" || engine.GRPC.MaxSendMessageBytes != 64<<20 { + t.Fatal("process defaults changed") } - if loaded.Mode != ModeEngine || loaded.Storage.MongoDB.URI != "mongodb://configured:27017" { - t.Fatalf("Load() = %#v", loaded) + gateway, err := Decode(strings.NewReader(gatewayConfig), nil) + if err != nil || gateway.Service.Request.MaxOperations != 1000 || gateway.Service.Request.Timeout != 0 { + t.Fatalf("Gateway defaults: %v", err) } -} - -func TestExampleConfigurationFilesLoad(t *testing.T) { - paths := []string{ - "../../configs/gateway.yaml", - "../../configs/engine.yaml", - "../../configs/worker.yaml", - "../../examples/quickstart/engine.yaml", "../../examples/quickstart/worker.yaml", "../../examples/quickstart/gateway.yaml", - "../../examples/kubernetes/sink.yaml", - } - for _, path := range paths { - t.Run(filepath.Base(path), func(t *testing.T) { - _, err := Load(path) - if err != nil { - t.Fatalf("Load(%q) error = %v", path, err) - } - }) - } -} - -func writeConfig(t *testing.T, contents string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "config.yaml") - err := os.WriteFile(path, []byte(contents), 0o600) + worker, err := Decode(strings.NewReader("mode: worker\nconsumer: {group_id: workers}\n"), strings.NewReader(kafkaStore)) if err != nil { - t.Fatalf("write config: %v", err) - } - return path -} - -func TestPrometheusRequiresExplicitOptIn(t *testing.T) { - tests := []struct { - name string - yaml string - enabled bool - address string + t.Fatal(err) + } + kafka := worker.Storage.Kafka + if kafka.DeadLetter.Topic != "mutations.dlq" || kafka.Consumer.MaxPollRecords != 500 || kafka.Consumer.ProcessingTimeout != 20*time.Second || kafka.Consumer.Retry.MaxAttempts != 10 { + t.Fatal("consumer defaults changed") + } +} + +func TestStoreIdentityAndBackendValidation(t *testing.T) { + for _, shared := range []string{ + "name: ''\nstorage: {driver: mongodb}", "name: INVALID\nstorage: {driver: mongodb}", "name: 'a/b'\nstorage: {driver: mongodb}", + "name: primary\nstorage: {driver: unknown}", "name: primary\nstorage: {driver: mongodb}", + "name: primary\nstorage: {driver: opensearch, search: {endpoints: []}}", + "name: primary\nstorage: {driver: opensearch, search: {endpoints: [http://search:9200], username: test}}", + "name: primary\nstorage: {driver: opensearch, search: {endpoints: [http://search:9200], username: test, password: test, api_key: test}}", + minimalStorage + "consumer: {group_id: workers}\n", minimalStorage + "producer: {}\n", + strings.Replace(minimalStorage, "uri: mongodb://127.0.0.1:1", "uri: mongodb://127.0.0.1:1\n max_concurrent_writes: 3", 1), + } { + loaded, err := Decode(strings.NewReader("mode: engine\n"), strings.NewReader(shared)) + if err == nil || !reflect.ValueOf(loaded).IsZero() { + t.Fatalf("invalid Store accepted: %v", err) + } + } + for _, driver := range []string{"elasticsearch", "opensearch"} { + shared := "name: search\nstorage:\n driver: " + driver + "\n search: {endpoints: [http://search:9200], username: test, password: test}\n" + loaded, err := Decode(strings.NewReader("mode: engine\n"), strings.NewReader(shared)) + if err != nil || loaded.Storage.Search.Username != "test" { + t.Fatalf("search configuration: %v", err) + } + _, err = Decode(strings.NewReader("mode: engine\nexecution:\n mongodb: {max_concurrent_writes: 3}\n"), strings.NewReader(shared)) + if err == nil { + t.Fatal("irrelevant MongoDB tuning accepted") + } + } +} + +func TestStrictDocumentsAndRequiredPaths(t *testing.T) { + cases := []struct { + component, store string + want string }{ - {name: "omitted", address: ":9090"}, - {name: "address only", yaml: "prometheus: {address: '127.0.0.1:9999'}\n", address: "127.0.0.1:9999"}, - {name: "disabled with address", yaml: "prometheus: {enabled: false, address: '127.0.0.1:9999'}\n", address: "127.0.0.1:9999"}, - {name: "enabled default address", yaml: "prometheus: {enabled: true}\n", enabled: true, address: ":9090"}, - {name: "enabled custom address", yaml: "prometheus: {enabled: true, address: ' 127.0.0.1:9999 '}\n", enabled: true, address: "127.0.0.1:9999"}, - {name: "empty address uses default", yaml: "prometheus: {enabled: true, address: ''}\n", enabled: true, address: ":9090"}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loaded, err := Decode(strings.NewReader(minimalStorage + test.yaml)) - if err != nil { - t.Fatal(err) - } - if loaded.Prometheus.Enabled != test.enabled || loaded.Prometheus.Address != test.address || loaded.Health.Address != ":8081" { - t.Fatalf("unexpected health/Prometheus settings: %#v / %#v", loaded.Health, loaded.Prometheus) - } - }) + {"mode: engine", "", "--store-config"}, {"mode: worker", "", "--store-config"}, {gatewayConfig, minimalStorage, "must not use"}, + {"mode: server", "", "mode is required"}, {"", "", "EOF"}, + {"mode: engine\nunknown: true", minimalStorage, "field unknown"}, + {"mode: engine\nmode: worker", minimalStorage, "already defined"}, + {"mode: engine\n---\nmode: engine", minimalStorage, "multiple YAML"}, + {"mode: engine", minimalStorage + "---\nname: other", "multiple YAML"}, + {"mode: engine", minimalStorage + "name: other", "already defined"}, + {strings.Repeat(" ", MaxFileBytes+1), "", "exceeds 4 MiB"}, + {"mode: engine", strings.Repeat(" ", MaxFileBytes+1), "exceeds 4 MiB"}, + } + for _, test := range cases { + var store io.Reader + if test.store != "" { + store = strings.NewReader(test.store) + } + loaded, err := Decode(strings.NewReader(test.component), store) + if err == nil || !strings.Contains(err.Error(), test.want) || !reflect.ValueOf(loaded).IsZero() { + t.Fatalf("want %q, got %v", test.want, err) + } + } + if _, err := Load("/nonexistent/component", ""); err == nil { + t.Fatal("missing component accepted") + } + if _, err := Load(writeConfig(t, "mode: engine"), "/nonexistent/store"); err == nil { + t.Fatal("missing Store accepted") + } +} + +func TestConfigurationExamples(t *testing.T) { + for _, mode := range []string{"gateway", "engine", "worker"} { + storePath := "" + if mode != "gateway" { + storePath = "../../configs/stores/primary.yaml" + } + if _, err := Load("../../configs/"+mode+".yaml", storePath); err != nil { + t.Fatal(err) + } } } diff --git a/internal/config/file.go b/internal/config/file.go index 68d495c..a7f7705 100644 --- a/internal/config/file.go +++ b/internal/config/file.go @@ -5,15 +5,19 @@ import "time" // File types preserve omission so defaults can depend on other configured limits. // Runtime code receives only resolved values through Config. type configFile struct { - Logging loggingFile `yaml:"logging"` - Memory memoryFile `yaml:"memory"` - Storage *storageFile `yaml:"storage"` - Gateway *gatewayFile `yaml:"gateway"` + Logging loggingFile `yaml:"logging"` + Memory memoryFile `yaml:"memory"` + + Gateway *gatewayFile `yaml:"forwarding"` Mode Mode `yaml:"mode"` - GRPC gRPCFile `yaml:"grpc"` + GRPC *gRPCFile `yaml:"grpc"` Health healthFile `yaml:"health"` Prometheus prometheusFile `yaml:"prometheus"` - Service serviceFile `yaml:"service"` + Request *requestFile `yaml:"request"` + Execution *executionFile `yaml:"execution"` + Batching *batchingFile `yaml:"batching"` + Producer *producerFile `yaml:"producer"` + Consumer *consumerFile `yaml:"consumer"` ShutdownTimeout *time.Duration `yaml:"shutdown_timeout"` } @@ -32,42 +36,27 @@ type prometheusFile struct { Address string `yaml:"address"` } -type serviceFile struct { - Request requestFile `yaml:"request"` - Execution executionFile `yaml:"execution"` - Publish publishFile `yaml:"publish"` - Batching batchingFile `yaml:"batching"` - Merge mergeFile `yaml:"merge"` -} - type requestFile struct { - Timeout *time.Duration `yaml:"timeout"` - MaxOperations *int `yaml:"max_operations"` - MaxReadBytes *byteSize `yaml:"max_read_bytes"` + MaxOperations *int `yaml:"max_operations"` } type executionFile struct { - MaxRequests *int `yaml:"max_requests"` - MaxBytes *byteSize `yaml:"max_bytes"` - Queue admissionQueueFile `yaml:"queue"` - Scan scanFile `yaml:"scan"` + MaxSnapshotBytes *byteSize `yaml:"max_snapshot_bytes"` + MaxOutputBytes *byteSize `yaml:"max_output_bytes"` + MongoDB mongoExecutionFile `yaml:"mongodb"` + Merge mergeFile `yaml:"merge"` } -type admissionQueueFile struct { - MaxRequests *int `yaml:"max_requests"` - MaxBytes *byteSize `yaml:"max_bytes"` - MaxWait *time.Duration `yaml:"max_wait"` -} - -type scanFile struct { - MaxRequests *int `yaml:"max_requests"` - MaxBytes *byteSize `yaml:"max_bytes"` - AdmissionWait *time.Duration `yaml:"admission_wait"` +type mongoExecutionFile struct { + MaxConcurrentWrites *int `yaml:"max_concurrent_writes"` + MaxConcurrentGroups *int `yaml:"max_concurrent_groups"` } -type publishFile struct { - MaxRequests *int `yaml:"max_requests"` - MaxBytes *byteSize `yaml:"max_bytes"` +// Store files contain shared identity and dependency policy, never role tuning. +type storeFile struct { + Name string `yaml:"name"` + Storage storageFile `yaml:"storage"` + Kafka kafkaFile `yaml:"kafka"` } type batchingFile struct { @@ -96,19 +85,15 @@ type luaFile struct { } type storageFile struct { - Name string `yaml:"name"` Driver Driver `yaml:"driver"` MongoDB mongoDBFile `yaml:"mongodb"` Search searchFile `yaml:"search"` - Kafka kafkaFile `yaml:"kafka"` } type mongoDBFile struct { - URI *string `yaml:"uri"` - URIFile *string `yaml:"uri_file"` - MetadataField string `yaml:"metadata_field"` - MaxConcurrentWrites *int `yaml:"max_concurrent_writes"` - MaxConcurrentGroups *int `yaml:"max_concurrent_groups"` + URI *string `yaml:"uri"` + URIFile *string `yaml:"uri_file"` + MetadataField string `yaml:"metadata_field"` } type searchFile struct { @@ -122,11 +107,10 @@ type searchFile struct { } type kafkaFile struct { - Enabled bool `yaml:"enabled"` - Brokers []string `yaml:"brokers"` - Topic topicFile `yaml:"topic"` - Producer producerFile `yaml:"producer"` - Consumer consumerFile `yaml:"consumer"` + Enabled bool `yaml:"enabled"` + Brokers []string `yaml:"brokers"` + Topic topicFile `yaml:"topic"` + DeadLetter deadLetterFile `yaml:"dead_letter"` } @@ -162,14 +146,12 @@ type deadLetterFile struct { } type gatewayFile struct { - MaxRequestsPerStore *int `yaml:"max_requests_per_store"` - DNSRefreshInterval *time.Duration `yaml:"dns_refresh_interval"` - Routes []Route `yaml:"routes"` - IdleTimeout *time.Duration `yaml:"idle_timeout"` - MaxConnections *int `yaml:"max_connections"` - MaxRequests *int `yaml:"max_requests"` - MaxBytes *byteSize `yaml:"max_bytes"` - MaxFanout *int `yaml:"max_fanout"` + DNSRefreshInterval *time.Duration `yaml:"dns_refresh_interval"` + Routes []Route `yaml:"routes"` + IdleTimeout *time.Duration `yaml:"idle_timeout"` + MaxConnections *int `yaml:"max_connections"` + + MaxFanout *int `yaml:"max_fanout"` } type memoryFile struct { diff --git a/internal/config/gateway.go b/internal/config/gateway.go index ed200e9..68cdc43 100644 --- a/internal/config/gateway.go +++ b/internal/config/gateway.go @@ -9,14 +9,14 @@ func resolveGateway(file gatewayFile, v *validator) Gateway { var loaded Gateway loaded.Routes = file.Routes if len(loaded.Routes) == 0 || len(loaded.Routes) > 10000 { - v.reject(errors.New("gateway.routes must contain between 1 and 10000 entries")) + v.reject(errors.New("forwarding.routes must contain between 1 and 10000 entries")) } - loaded.DNSRefreshInterval = v.duration("gateway.dns_refresh_interval", file.DNSRefreshInterval, 30*time.Second) - loaded.IdleTimeout = v.duration("gateway.idle_timeout", file.IdleTimeout, 5*time.Minute) - loaded.MaxConnections = v.bounded("gateway.max_connections", file.MaxConnections, 256, 10000) - loaded.MaxRequests = v.bounded("gateway.max_requests", file.MaxRequests, 128, 10000) - loaded.MaxRequestsPerStore = v.bounded("gateway.max_requests_per_store", file.MaxRequestsPerStore, min(32, loaded.MaxRequests), loaded.MaxRequests) - loaded.MaxBytes = v.bytes("gateway.max_bytes", file.MaxBytes, 256<<20, 1<<40) - loaded.MaxFanout = v.bounded("gateway.max_fanout", file.MaxFanout, 8, 128) + loaded.DNSRefreshInterval = v.duration("forwarding.dns_refresh_interval", file.DNSRefreshInterval, 30*time.Second) + loaded.IdleTimeout = v.duration("forwarding.idle_timeout", file.IdleTimeout, 5*time.Minute) + loaded.MaxConnections = v.bounded("forwarding.max_connections", file.MaxConnections, 256, 10000) + loaded.MaxRequests = v.bounded("forwarding.max_requests", nil, 128, 10000) + loaded.MaxRequestsPerStore = v.bounded("forwarding.max_requests_per_store", nil, min(32, loaded.MaxRequests), loaded.MaxRequests) + loaded.MaxBytes = v.bytes("forwarding.max_bytes", nil, 256<<20, 1<<40) + loaded.MaxFanout = v.bounded("forwarding.max_fanout", file.MaxFanout, 8, 128) return loaded } diff --git a/internal/config/kafka.go b/internal/config/kafka.go index f6e824c..d814ec5 100644 --- a/internal/config/kafka.go +++ b/internal/config/kafka.go @@ -17,8 +17,8 @@ func resolveKafka(prefix string, file kafkaFile, v *validator) Kafka { topic.ReplicationFactor = v.bounded(prefix+".topic.replication_factor", file.Topic.ReplicationFactor, 2, 1<<15-1) topic.Retention = v.duration(prefix+".topic.retention", file.Topic.Retention, 72*time.Hour) topic.MinInSyncReplicas = v.bounded(prefix+".topic.min_insync_replicas", file.Topic.MinInSyncReplicas, min(2, topic.ReplicationFactor), topic.ReplicationFactor) - loaded.Producer.MaxBufferedBytes = v.bytes(prefix+".producer.max_buffered_bytes", file.Producer.MaxBufferedBytes, 64<<20, 1<<30) - topic.MaxRecordBytes = v.bytes(prefix+".topic.max_record_bytes", file.Topic.MaxRecordBytes, 900<<10, min(64<<20, loaded.Producer.MaxBufferedBytes)) + + topic.MaxRecordBytes = v.bytes(prefix+".topic.max_record_bytes", file.Topic.MaxRecordBytes, 900<<10, 64<<20) loaded.DeadLetter.Topic = strings.TrimSpace(file.DeadLetter.Topic) if topic.Name != "" && loaded.DeadLetter.Topic == "" { loaded.DeadLetter.Topic = topic.Name + ".dlq" @@ -30,23 +30,39 @@ func resolveKafka(prefix string, file kafkaFile, v *validator) Kafka { if loaded.DeadLetter.Retention < time.Millisecond { v.reject(fmt.Errorf("%s.dead_letter.retention must be at least 1ms", prefix)) } - consumer := &loaded.Consumer - consumer.GroupID = strings.TrimSpace(file.Consumer.GroupID) - consumer.MaxPollRecords = v.integer(prefix+".consumer.max_poll_records", file.Consumer.MaxPollRecords, 500) - consumer.ProcessingTimeout = v.duration(prefix+".consumer.processing_timeout", file.Consumer.ProcessingTimeout, 20*time.Second) + return loaded +} + +func resolveProducer(file *producerFile, v *validator) Producer { + if file == nil { + file = &producerFile{} + } + loaded := Producer{MaxBufferedBytes: v.bytes("producer.max_buffered_bytes", file.MaxBufferedBytes, 64<<20, 1<<30)} + return loaded +} + +func resolveConsumer(file *consumerFile, v *validator) Consumer { + if file == nil { + file = &consumerFile{} + } + var loaded Consumer + consumer := &loaded + consumer.GroupID = strings.TrimSpace(file.GroupID) + consumer.MaxPollRecords = v.integer("consumer.max_poll_records", file.MaxPollRecords, 500) + consumer.ProcessingTimeout = v.duration("consumer.processing_timeout", file.ProcessingTimeout, 20*time.Second) if consumer.ProcessingTimeout > 20*time.Second { - v.reject(fmt.Errorf("%s.consumer.processing_timeout must not exceed 20s", prefix)) + v.reject(errors.New("consumer.processing_timeout must not exceed 20s")) } - consumer.Retry.MaxAttempts = v.integer(prefix+".consumer.retry.max_attempts", file.Consumer.Retry.MaxAttempts, 10) - consumer.Retry.Backoff = v.duration(prefix+".consumer.retry.backoff", file.Consumer.Retry.Backoff, 100*time.Millisecond) - consumer.Retry.MaxBackoff = v.duration(prefix+".consumer.retry.max_backoff", file.Consumer.Retry.MaxBackoff, 10*time.Second) + consumer.Retry.MaxAttempts = v.integer("consumer.retry.max_attempts", file.Retry.MaxAttempts, 10) + consumer.Retry.Backoff = v.duration("consumer.retry.backoff", file.Retry.Backoff, 100*time.Millisecond) + consumer.Retry.MaxBackoff = v.duration("consumer.retry.max_backoff", file.Retry.MaxBackoff, 10*time.Second) if consumer.Retry.MaxBackoff < consumer.Retry.Backoff { - v.reject(fmt.Errorf("%s.consumer.retry.max_backoff must be at least %s.consumer.retry.backoff", prefix, prefix)) + v.reject(errors.New("consumer.retry.max_backoff must be at least consumer.retry.backoff")) } // Worker retries double their delay before capping it. Reject durations // whose doubling would overflow into an immediate retry. if consumer.Retry.MaxBackoff > time.Duration(1<<63-1)/2 { - v.reject(fmt.Errorf("%s.consumer.retry.max_backoff is too large to double safely", prefix)) + v.reject(errors.New("consumer.retry.max_backoff is too large to double safely")) } return loaded } @@ -55,18 +71,18 @@ func validateKafkaResources(loaded Config) error { kafka := loaded.Storage.Kafka if !kafka.Enabled { if loaded.Mode == ModeWorker { - return errors.New("worker requires storage.kafka.enabled") + return errors.New("worker requires kafka.enabled") } return nil } if len(kafka.Brokers) == 0 || kafka.Topic.Name == "" { - return errors.New("storage.kafka.brokers and storage.kafka.topic.name are required") + return errors.New("kafka.brokers and kafka.topic.name are required") } if loaded.Mode == ModeWorker && kafka.Consumer.GroupID == "" { - return errors.New("storage.kafka.consumer.group_id is required in worker mode") + return errors.New("consumer.group_id is required in worker mode") } if kafka.DeadLetter.Topic == kafka.Topic.Name { - return errors.New("storage.kafka.dead_letter.topic must differ from topic.name") + return errors.New("kafka.dead_letter.topic must differ from topic.name") } return nil } diff --git a/internal/config/limits_test.go b/internal/config/limits_test.go index bddce4b..f9a443d 100644 --- a/internal/config/limits_test.go +++ b/internal/config/limits_test.go @@ -1,116 +1,61 @@ package config import ( - "fmt" - "reflect" "strings" "testing" - "time" ) -const minimalStorage = `mode: engine -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://127.0.0.1:1 -` - -func TestDecodeDerivesLimitsFromTheirOwners(t *testing.T) { - input := minimalStorage + `grpc: - max_send_message_bytes: 2097152 - max_receive_message_bytes: 268435456 -service: - request: - timeout: 1s - max_operations: 15000 - execution: - max_requests: 8 - max_bytes: 67108864 - publish: - max_requests: 7 -` - loaded, err := Decode(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } - service := loaded.Service - if service.Request.MaxReadBytes != 1<<20 || service.Execution.Scan.MaxRequests != 4 || - service.Execution.Scan.MaxBytes != 32<<20 || - service.Execution.Scan.AdmissionWait != time.Second { - t.Fatalf("dependent request and execution defaults: %+v", service) - } - if service.Batching.MaxOperations != 15000 || service.Batching.Queue.MaxOperations != 15000 || service.Batching.Queue.MaxBytes != 256<<20 { - t.Fatalf("dependent batching defaults: %+v", service.Batching) - } - if service.Publish.MaxRequests != 7 || service.Publish.MaxBytes != 256<<20 { - t.Fatalf("execution limits leaked into publication: %+v", service.Publish) - } - if service.Execution.Queue.MaxWait != time.Second || service.Execution.Queue.MaxRequests != 1024 || service.Execution.Queue.MaxBytes != 32<<20 { - t.Fatalf("invalid direct admission defaults: %+v", service.Execution.Queue) +func TestExecutionBudgetsAreIndependentOfTransport(t *testing.T) { + for _, input := range []string{ + "mode: engine\ngrpc: {max_send_message_bytes: 1MiB}\nexecution: {max_snapshot_bytes: 64MiB, max_output_bytes: 128MiB}\n", + "mode: worker\nconsumer: {group_id: workers}\nexecution: {max_snapshot_bytes: 64MiB, max_output_bytes: 128MiB}\n", + } { + loaded, err := Decode(strings.NewReader(input), strings.NewReader(kafkaStore)) + if err != nil || loaded.Service.Execution.MaxSnapshotBytes != 64<<20 || loaded.Service.Execution.MaxOutputBytes != 128<<20 { + t.Fatalf("execution depends on gRPC: %v", err) + } } } -func TestDecodeRejectsInvalidResourceLimitsWithoutReturningPartialConfig(t *testing.T) { - cases := []struct { - section string - field string - values []string - }{ - {section: "request", field: "timeout", values: []string{"0s", "-1ms", "301s", "9223372036854775808ns", "30"}}, - {section: "request", field: "max_read_bytes", values: []string{"0", "-1", "33554433"}}, - {section: "execution", field: "max_requests", values: []string{"0", "-1", "10001"}}, - {section: "execution", field: "max_bytes", values: []string{"0", "-1", "18253611008"}}, - {section: "execution:\n queue", field: "max_requests", values: []string{"0", "10001"}}, - {section: "execution:\n queue", field: "max_bytes", values: []string{"0", "17GiB"}}, - {section: "execution:\n queue", field: "max_wait", values: []string{"0s", "-1s", "30.001s"}}, - {section: "execution:\n scan", field: "max_requests", values: []string{"0", "129"}}, - {section: "execution:\n scan", field: "max_bytes", values: []string{"0", "268435457"}}, - {section: "execution:\n scan", field: "admission_wait", values: []string{"0s", "-1s", "30.001s"}}, - {section: "publish", field: "max_requests", values: []string{"0", "-1", "10001"}}, - {section: "publish", field: "max_bytes", values: []string{"0", "-1", "18253611008"}}, +func TestRejectInvalidRoleLimits(t *testing.T) { + for _, fields := range []string{ + "execution: {max_snapshot_bytes: 0}", "execution: {max_output_bytes: -1}", "execution: {mongodb: {max_concurrent_groups: 0}}", + "execution: {merge: {max_attempts: 0}}", "execution: {merge: {lua: {timeout: 0s}}}", "execution: {merge: {lua: {max_source_bytes: 0}}}", + "batching: {max_wait: 0s}", "batching: {max_operations: 0}", "batching: {max_operations: 20, queue: {max_operations: 19}}", + "batching: {queue: {max_bytes: 1MiB}}", "producer: {max_buffered_bytes: 2GiB}", "producer: {max_buffered_bytes: 1KiB}", + } { + if _, err := Decode(strings.NewReader("mode: engine\n"+fields), strings.NewReader(kafkaStore)); err == nil { + t.Fatalf("invalid limits accepted: %s", fields) + } } - for _, test := range cases { - for _, value := range test.values { - t.Run(test.section+"/"+test.field+"/"+value, func(t *testing.T) { - indent := " " - if strings.Contains(test.section, "\n") { - indent += " " - } - input := fmt.Sprintf("%sservice:\n %s:\n%s%s: %s\n", minimalStorage, test.section, indent, test.field, value) - loaded, err := Decode(strings.NewReader(input)) - if err == nil || !reflect.ValueOf(loaded).IsZero() { - t.Fatalf("invalid configuration escaped validation: config=%+v error=%v", loaded, err) - } - }) + for _, fields := range []string{ + "processing_timeout: 21s", "max_poll_records: 0", "processing_timeout: 0s", "processing_timeout: 20", + "retry: {max_backoff: 4611686018427387904ns}", "retry: {backoff: 2s, max_backoff: 1s}", "retry: {max_attempts: 0}", + } { + input := "mode: worker\nconsumer:\n group_id: workers\n " + fields + if _, err := Decode(strings.NewReader(input), strings.NewReader(kafkaStore)); err == nil { + t.Fatalf("invalid consumer accepted: %s", fields) } } } -func TestDecodeRejectsRemovedConfigurationPaths(t *testing.T) { - for _, field := range []string{"max_in_flight_requests: 10", "max_publish_bytes: 1024", "max_operations: 10", "lua: {}", "store_execution_bytes: {}"} { - _, err := Decode(strings.NewReader(minimalStorage + "service:\n " + field + "\n")) - if err == nil || !strings.Contains(err.Error(), "field ") { - t.Fatalf("old service field was silently accepted: %s, %v", field, err) +func TestKafkaSharedPolicyValidation(t *testing.T) { + for _, kafka := range []string{ + "enabled: true", "enabled: true\n brokers: [localhost:1]", "topic: {retention: 1ns}", "dead_letter: {retention: 1ns}", + "topic: {partitions: 0}", "topic: {replication_factor: 1, min_insync_replicas: 2}", "topic: {max_record_bytes: 65MiB}", + "enabled: true\n brokers: [localhost:1]\n topic: {name: a}\n dead_letter: {topic: a}", + "consumer: {}", "producer: {}", + } { + if _, err := Decode(strings.NewReader("mode: engine"), strings.NewReader(minimalStorage+"kafka:\n "+kafka)); err == nil { + t.Fatalf("invalid Kafka policy accepted: %s", kafka) } } - _, err := Decode(strings.NewReader(minimalStorage + "shutdown_timeout_seconds: 15\n")) - if err == nil || !strings.Contains(err.Error(), "shutdown_timeout_seconds") { - t.Fatalf("old duration field was silently accepted: %v", err) - } -} - -func TestDecodeRejectsKafkaDurationTruncationAndOverflow(t *testing.T) { - for _, fragment := range []string{ - "topic:\n retention: 1ns", - "dead_letter:\n retention: 1ns", - "consumer:\n processing_timeout: 21s", - "consumer:\n retry:\n max_backoff: 4611686018427387904ns", - "consumer:\n retry:\n backoff: 2s\n max_backoff: 1s", + for _, test := range []struct{ component, store string }{ + {"mode: worker\nconsumer: {group_id: workers}", minimalStorage}, + {"mode: worker", kafkaStore}, {"mode: engine\nproducer: {}", minimalStorage}, } { - _, err := Decode(strings.NewReader(minimalStorage + " kafka:\n " + fragment + "\n")) - if err == nil || !strings.Contains(err.Error(), "storage.kafka.") { - t.Fatalf("invalid Kafka duration accepted: %s, %v", fragment, err) + if _, err := Decode(strings.NewReader(test.component), strings.NewReader(test.store)); err == nil { + t.Fatal("invalid role/Kafka combination accepted") } } } diff --git a/internal/config/load.go b/internal/config/load.go index c195c40..a541ad9 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -7,64 +7,94 @@ import ( "io" "math" "os" - "reflect" "strings" "time" "unicode/utf8" + "github.com/liran/sink-go/uri" "github.com/liran/sink/internal/capacity" "gopkg.in/yaml.v3" ) const MaxFileBytes = 4 << 20 -// Load reads one strict YAML document. It never connects to storage or Kafka. -func Load(path string) (Config, error) { +// Load reads separate strict component and Store YAML documents. It never connects to storage or Kafka. +func Load(componentPath, storePath string) (Config, error) { var empty Config - file, err := os.Open(path) + component, err := os.Open(componentPath) if err != nil { - return empty, fmt.Errorf("read config file %q: %w", path, err) + return empty, fmt.Errorf("read component config %q: %w", componentPath, err) } - defer file.Close() - loaded, err := Decode(file) + defer component.Close() + var store io.Reader + if storePath != "" { + file, err := os.Open(storePath) + if err != nil { + return empty, fmt.Errorf("read Store config %q: %w", storePath, err) + } + defer file.Close() + store = file + } + return Decode(component, store) +} + +// Decode validates separate standard YAML documents before opening dependencies. +// Store is required for Engine/Worker and forbidden for Gateway. +func Decode(component io.Reader, store io.Reader) (Config, error) { + var empty Config + var file configFile + if err := decodeDocument(component, &file); err != nil { + return empty, fmt.Errorf("component config: %w", err) + } + if file.Mode != ModeGateway && file.Mode != ModeEngine && file.Mode != ModeWorker { + return empty, errors.New("mode is required and must be gateway, engine, or worker") + } + if file.Mode == ModeGateway && store != nil { + return empty, errors.New("gateway must not use --store-config") + } + if file.Mode != ModeGateway && store == nil { + return empty, errors.New("engine and worker require --store-config") + } + var shared storeFile + if store != nil { + if err := decodeDocument(store, &shared); err != nil { + return empty, fmt.Errorf("store config: %w", err) + } + } + loaded, err := resolve(file, shared) if err != nil { - return empty, fmt.Errorf("config file %q: %w", path, err) + return empty, err } return loaded, nil } -// Decode resolves defaults and validates the complete configuration before it -// becomes available to the application. Explicit zero limits are invalid. -func Decode(reader io.Reader) (Config, error) { - var empty Config +func decodeDocument(reader io.Reader, target any) error { + if reader == nil { + return errors.New("configuration reader is required") + } data, err := io.ReadAll(io.LimitReader(reader, MaxFileBytes+1)) if err != nil { - return empty, fmt.Errorf("read configuration: %w", err) + return fmt.Errorf("read configuration: %w", err) } if len(data) > MaxFileBytes { - return empty, errors.New("configuration exceeds 4 MiB") + return errors.New("configuration exceeds 4 MiB") } - var file configFile decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder.KnownFields(true) - if err := decoder.Decode(&file); err != nil { - return empty, fmt.Errorf("decode configuration: %w", err) + if err := decoder.Decode(target); err != nil { + return fmt.Errorf("decode configuration: %w", err) } var extra any if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { if err != nil { - return empty, fmt.Errorf("decode configuration: %w", err) + return fmt.Errorf("decode configuration: %w", err) } - return empty, errors.New("multiple YAML documents are not supported") + return errors.New("multiple YAML documents are not supported") } - loaded, err := resolve(file) - if err != nil { - return empty, err - } - return loaded, nil + return nil } -func resolve(file configFile) (Config, error) { +func resolve(file configFile, shared storeFile) (Config, error) { var loaded Config loaded.Mode = file.Mode if loaded.Mode != ModeWorker && loaded.Mode != ModeEngine && loaded.Mode != ModeGateway { @@ -80,50 +110,77 @@ func resolve(file configFile) (Config, error) { } loaded.Memory.BurstPercent = v.bounded("memory.burst_percent", file.Memory.BurstPercent, capacity.DefaultBurstPercent, 99) loaded.Memory.WaitTimeout = v.duration("memory.wait_timeout", file.Memory.WaitTimeout, 2*time.Second) - loaded.GRPC.Address = valueOrDefault(file.GRPC.Address, ":8080") - loaded.GRPC.MaxReceiveMessageBytes = v.bytes("grpc.max_receive_message_bytes", file.GRPC.MaxReceiveMessageBytes, 64<<20, math.MaxInt) - loaded.GRPC.MaxSendMessageBytes = v.bytes("grpc.max_send_message_bytes", file.GRPC.MaxSendMessageBytes, 64<<20, math.MaxInt) + grpcFile := gRPCFile{} + if file.GRPC != nil { + grpcFile = *file.GRPC + } + loaded.GRPC.Address = valueOrDefault(grpcFile.Address, ":8080") + loaded.GRPC.MaxReceiveMessageBytes = v.bytes("grpc.max_receive_message_bytes", grpcFile.MaxReceiveMessageBytes, 64<<20, math.MaxInt) + loaded.GRPC.MaxSendMessageBytes = v.bytes("grpc.max_send_message_bytes", grpcFile.MaxSendMessageBytes, 64<<20, math.MaxInt) loaded.Prometheus.Enabled = file.Prometheus.Enabled loaded.Health.Address = valueOrDefault(file.Health.Address, ":8081") loaded.Prometheus.Address = valueOrDefault(file.Prometheus.Address, ":9090") loaded.ShutdownTimeout = v.duration("shutdown_timeout", file.ShutdownTimeout, 15*time.Second) - loaded.Service = resolveService(file.Service, loaded.GRPC, &v) + loaded.Service = resolveService(file, loaded.GRPC, &v) + if loaded.Mode == ModeWorker && (file.Memory.BurstPercent != nil || file.Memory.WaitTimeout != nil) { + v.reject(errors.New("worker memory only accepts max_bytes")) + } if v.err != nil { return loaded, v.err } if loaded.Mode == ModeGateway { - for _, section := range []any{file.Service.Execution, file.Service.Publish, file.Service.Batching, file.Service.Merge} { - if !reflect.ValueOf(section).IsZero() { - return loaded, errors.New("gateway only accepts service.request; execution, publish, batching and merge belong to Engine/Worker") - } - } - if file.Storage != nil { - return loaded, errors.New("gateway must not configure storage") + if file.Execution != nil || file.Producer != nil || file.Consumer != nil || file.Batching != nil { + return loaded, errors.New("gateway must not configure execution, producer, consumer or batching") } if file.Gateway == nil { - return loaded, errors.New("gateway configuration is required") + return loaded, errors.New("forwarding configuration is required") } loaded.Gateway = resolveGateway(*file.Gateway, &v) return loaded, v.err } - if file.Gateway != nil { - return loaded, errors.New("gateway configuration requires gateway mode") + if file.Request != nil || file.Gateway != nil { + return loaded, errors.New("request and forwarding require gateway mode") + } + if loaded.Mode == ModeEngine && file.Consumer != nil { + return loaded, errors.New("consumer requires worker mode") } - if file.Storage == nil { - return loaded, errors.New("engine and worker require singular storage with name") + if loaded.Mode == ModeWorker && (file.Producer != nil || file.Batching != nil || file.GRPC != nil) { + return loaded, errors.New("worker must not configure producer, batching or grpc") } - name := file.Storage.Name + name := shared.Name if name == "" || len(name) > 256 || !utf8.ValidString(name) || strings.TrimSpace(name) != name || strings.ContainsAny(name, "\x00\r\n\t") { - return loaded, errors.New("storage.name must be a nonempty valid identity of at most 256 bytes") + return loaded, errors.New("store name must be a nonempty valid identity of at most 256 bytes") } - configured, err := resolveStorage("storage", *file.Storage) + configured, err := resolveStorage("storage", shared.Storage) if err != nil { return loaded, err } + configured.Name = name + if !uri.ValidStore(name) { + return loaded, errors.New("store name must be a canonical lowercase identity") + } + configured.MongoDB.MaxConcurrentWrites = loaded.Service.Execution.MongoDB.MaxConcurrentWrites + configured.MongoDB.MaxConcurrentGroups = loaded.Service.Execution.MongoDB.MaxConcurrentGroups + configured.Kafka = resolveKafka("kafka", shared.Kafka, &v) + configured.Kafka.Producer = resolveProducer(file.Producer, &v) + configured.Kafka.Consumer = resolveConsumer(file.Consumer, &v) + if file.Producer != nil && !configured.Kafka.Enabled { + v.reject(errors.New("producer requires Kafka enabled in Store config")) + } + if loaded.Mode == ModeEngine && configured.Kafka.Topic.MaxRecordBytes > configured.Kafka.Producer.MaxBufferedBytes { + v.reject(errors.New("producer.max_buffered_bytes must cover kafka.topic.max_record_bytes")) + } + if file.Execution != nil && file.Execution.MongoDB != (mongoExecutionFile{}) && configured.Driver != DriverMongoDB { + v.reject(errors.New("execution.mongodb requires the mongodb driver")) + } + if v.err != nil { + return loaded, v.err + } loaded.Storage = configured if err := validateKafkaResources(loaded); err != nil { return loaded, err } + return loaded, nil } diff --git a/internal/config/logging_test.go b/internal/config/logging_test.go index b6d1272..27aa7f7 100644 --- a/internal/config/logging_test.go +++ b/internal/config/logging_test.go @@ -6,10 +6,10 @@ import ( "time" ) -const loggingBase = "mode: engine\nstorage:\n name: primary\n driver: mongodb\n mongodb:\n uri: mongodb://localhost:27017\n" +const loggingBase = "mode: engine\n" func TestLoggingDefaults(t *testing.T) { - loaded, err := Decode(strings.NewReader(loggingBase)) + loaded, err := Decode(strings.NewReader(loggingBase), strings.NewReader(minimalStorage)) if err != nil { t.Fatal(err) } @@ -35,7 +35,7 @@ func TestLoggingRejectsInvalidSettings(t *testing.T) { } for _, fragment := range cases { t.Run(fragment, func(t *testing.T) { - _, err := Decode(strings.NewReader(loggingBase + "logging:\n " + fragment + "\n")) + _, err := Decode(strings.NewReader(loggingBase+"logging:\n "+fragment+"\n"), strings.NewReader(minimalStorage)) if err == nil { t.Fatal("accepted invalid logging configuration") } @@ -61,7 +61,7 @@ func TestLoggingExplicitOptions(t *testing.T) { export_timeout: 4s shutdown_timeout: 6s ` - loaded, err := Decode(strings.NewReader(yaml)) + loaded, err := Decode(strings.NewReader(yaml), strings.NewReader(minimalStorage)) if err != nil { t.Fatal(err) } diff --git a/internal/config/memory_test.go b/internal/config/memory_test.go index c91eacf..76f492f 100644 --- a/internal/config/memory_test.go +++ b/internal/config/memory_test.go @@ -7,8 +7,8 @@ import ( ) func TestMemoryConfiguration(t *testing.T) { - base := "mode: gateway\ngateway:\n routes:\n - store: primary\n target: 127.0.0.1:8080\n tls: {insecure: true}\n" - defaults, err := Decode(strings.NewReader(base)) + base := "mode: gateway\nforwarding:\n routes:\n - store: primary\n target: 127.0.0.1:8080\n tls: {insecure: true}\n" + defaults, err := Decode(strings.NewReader(base), nil) if err != nil { t.Fatal(err) } @@ -16,7 +16,7 @@ func TestMemoryConfiguration(t *testing.T) { t.Fatalf("defaults: %+v", defaults.Memory) } explicit := base + "memory: {max_bytes: 128MiB, burst_percent: 15, wait_timeout: 500ms}\n" - loaded, err := Decode(strings.NewReader(explicit)) + loaded, err := Decode(strings.NewReader(explicit), nil) if err != nil { t.Fatal(err) } @@ -24,7 +24,7 @@ func TestMemoryConfiguration(t *testing.T) { t.Fatalf("explicit: %+v", loaded.Memory) } for _, value := range []string{"burst_percent: 0", "burst_percent: 100", "burst_percent: -1", "max_bytes: 0", "max_bytes: 512", "wait_timeout: 0s", "unknown: 1"} { - if _, err := Decode(strings.NewReader(base + "memory: {" + value + "}\n")); err == nil { + if _, err := Decode(strings.NewReader(base+"memory: {"+value+"}\n"), nil); err == nil { t.Fatalf("accepted %s", value) } } diff --git a/internal/config/roles_test.go b/internal/config/roles_test.go index 4739d5d..8bd4932 100644 --- a/internal/config/roles_test.go +++ b/internal/config/roles_test.go @@ -5,76 +5,37 @@ import ( "testing" ) -func TestIsolatedRoles(t *testing.T) { - tests := []struct { - name, yaml string - valid bool - }{ - {"missing mode", "storage: {name: a}\n", false}, - {"removed server", "mode: server\n", false}, - {"removed all", "mode: all\n", false}, - {"gateway", "mode: gateway\ngateway:\n routes: [{store: a, target: 127.0.0.1:1}]\n", true}, - {"gateway storage", "mode: gateway\ngateway: {routes: [{store: a, target: 127.0.0.1:1}]}\nstorage: {name: a}\n", false}, - {"engine", "mode: engine\nstorage:\n name: a\n driver: mongodb\n mongodb: {uri: 'mongodb://localhost:27017'}\n", true}, - {"missing identity", "mode: engine\nstorage:\n driver: mongodb\n mongodb: {uri: 'mongodb://localhost:27017'}\n", false}, - {"engine plural", "mode: engine\nstorages:\n - name: a\n driver: mongodb\n mongodb: {uri: 'mongodb://localhost:27017'}\n", false}, - {"worker multiple", "mode: worker\nstorages: [{name: a}, {name: b}]\n", false}, - {"routes role", "mode: server\ngateway: {routes: [{store: a, target: 127.0.0.1:1}]}\n", false}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := Decode(strings.NewReader(test.yaml)) - if (err == nil) != test.valid { - t.Fatalf("Decode error=%v", err) +func TestRoleBoundariesAndRemovedFields(t *testing.T) { + for _, test := range []struct{ mode, fields string }{ + {"gateway", "consumer: {}"}, {"gateway", "producer: {}"}, {"gateway", "execution: {}"}, {"gateway", "batching: {}"}, + {"engine", "request: {}"}, {"engine", "consumer: {}"}, {"engine", "forwarding: {}"}, + {"worker", "request: {}"}, {"worker", "batching: {}"}, {"worker", "producer: {}"}, {"worker", "grpc: {}"}, {"worker", "grpc: {address: ':8080'}"}, + {"worker", "memory: {burst_percent: 10}"}, {"worker", "memory: {wait_timeout: 2s}"}, + {"gateway", "request: {timeout: 30s}"}, {"gateway", "request: {max_response_bytes: 32MiB}"}, {"gateway", "request: {max_read_bytes: 32MiB}"}, + {"engine", "service: {}"}, {"engine", "storage: {}"}, {"engine", "store_file: store.yaml"}, {"engine", "kafka: {}"}, + } { + t.Run(test.mode+"/"+test.fields, func(t *testing.T) { + base := "mode: " + test.mode + "\n" + store := strings.NewReader(kafkaStore) + if test.mode == "gateway" { + if _, err := Decode(strings.NewReader(gatewayConfig+test.fields), nil); err == nil { + t.Fatal("invalid Gateway section accepted") + } + return + } + if test.mode == "worker" { + base += "consumer: {group_id: workers}\n" + } + if _, err := Decode(strings.NewReader(base+test.fields), store); err == nil { + t.Fatal("invalid role section accepted") } }) } } -func TestRejectRemovedStoreSettings(t *testing.T) { - for _, extra := range []string{ - "service:\n execution:\n max_requests_per_store: 32\n", - "service:\n execution:\n queue:\n max_requests_per_store: 32\n", - "service:\n execution:\n scan:\n max_requests_per_store: 32\n", - "service:\n publish:\n max_requests_per_store: 32\n", - " database_id: removed-identity\n", - " limits:\n max_execution_bytes: 1MiB\n", - } { - _, err := Decode(strings.NewReader(minimalStorage + extra)) - if err == nil || !strings.Contains(err.Error(), "field ") { - t.Fatalf("removed Store setting accepted: %v", err) - } - } -} - -func TestRejectRemovedGatewaySettings(t *testing.T) { - for _, extra := range []string{ - " routes_file: routes.yaml\n", - " reload_interval: 5s\n", - " state: active\n", - } { - input := "mode: gateway\ngateway:\n routes:\n - store: primary\n target: 127.0.0.1:8080\n" + extra - _, err := Decode(strings.NewReader(input)) - if err == nil || !strings.Contains(err.Error(), "field ") { - t.Fatalf("removed Gateway setting accepted: %v", err) - } - } -} - -func TestConfigurationSizeIsBounded(t *testing.T) { - _, err := Decode(strings.NewReader(strings.Repeat(" ", MaxFileBytes+1))) - if err == nil || !strings.Contains(err.Error(), "exceeds 4 MiB") { - t.Fatalf("oversized configuration accepted: %v", err) - } -} - -func TestHealthAddressIsIndependentOfPrometheus(t *testing.T) { - input := minimalStorage + "health: {address: ' 127.0.0.1:8082 '}\nprometheus: {enabled: false, address: '127.0.0.1:9092'}\n" - loaded, err := Decode(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } - if loaded.Health.Address != "127.0.0.1:8082" || loaded.Prometheus.Address != "127.0.0.1:9092" || loaded.Prometheus.Enabled { - t.Fatalf("health and Prometheus settings are coupled: %#v / %#v", loaded.Health, loaded.Prometheus) +func TestHealthAndMetricsIndependent(t *testing.T) { + loaded, err := Decode(strings.NewReader(gatewayConfig+"health: {address: '127.0.0.1:8082'}\nprometheus: {enabled: false, address: '127.0.0.1:9092'}"), nil) + if err != nil || loaded.Health.Address != "127.0.0.1:8082" || loaded.Prometheus.Enabled || loaded.Prometheus.Address != "127.0.0.1:9092" { + t.Fatalf("listener defaults: %v", err) } } diff --git a/internal/config/secrets_test.go b/internal/config/secrets_test.go index 1b8364d..234301d 100644 --- a/internal/config/secrets_test.go +++ b/internal/config/secrets_test.go @@ -22,9 +22,9 @@ func TestCredentialFiles(t *testing.T) { } else { section = fmt.Sprintf("search:\n endpoints: [http://search:9200]\n username_file: %q\n password_file: %q", path, path) } - input := fmt.Sprintf("mode: engine\nstorage:\n name: test\n driver: %s\n %s\n", driver, section) + input := fmt.Sprintf("name: test\nstorage:\n driver: %s\n %s\n", driver, section) reader := strings.NewReader(input) - loaded, err := Decode(reader) + loaded, err := Decode(strings.NewReader("mode: engine"), reader) if err != nil { t.Fatal(err) } @@ -36,9 +36,9 @@ func TestCredentialFiles(t *testing.T) { t.Fatal("basic authentication bytes changed") } if driver != "mongodb" { - input = fmt.Sprintf("mode: engine\nstorage:\n name: test\n driver: %s\n search:\n endpoints: [http://search:9200]\n api_key_file: %q\n", driver, path) + input = fmt.Sprintf("name: test\nstorage:\n driver: %s\n search:\n endpoints: [http://search:9200]\n api_key_file: %q\n", driver, path) reader = strings.NewReader(input) - loaded, err = Decode(reader) + loaded, err = Decode(strings.NewReader("mode: engine"), reader) if err != nil || loaded.Storage.Search.APIKey != secret { t.Fatal("API key file was not resolved correctly") } @@ -81,9 +81,9 @@ func TestCredentialFileValidation(t *testing.T) { section, driver = "mongodb:\n", "mongodb" } // Even an explicitly empty literal must not silently lose to a file. - input := fmt.Sprintf("mode: engine\nstorage:\n name: test\n driver: %s\n %s %s: ''\n %s_file: %q\n", driver, section, field, field, path) + input := fmt.Sprintf("name: test\nstorage:\n driver: %s\n %s %s: ''\n %s_file: %q\n", driver, section, field, field, path) reader := strings.NewReader(input) - if _, err := Decode(reader); err == nil || !strings.Contains(err.Error(), "mutually exclusive") { + if _, err := Decode(strings.NewReader("mode: engine"), reader); err == nil || !strings.Contains(err.Error(), "mutually exclusive") { t.Fatal("ambiguous credential sources accepted") } } diff --git a/internal/config/service.go b/internal/config/service.go index b11e7f0..ac2469f 100644 --- a/internal/config/service.go +++ b/internal/config/service.go @@ -6,60 +6,51 @@ import ( "time" ) -func resolveService(file serviceFile, grpc GRPC, v *validator) Service { +func resolveService(file configFile, grpc GRPC, v *validator) Service { var loaded Service - request := &loaded.Request - request.Timeout = v.duration("service.request.timeout", file.Request.Timeout, 30*time.Second) - if request.Timeout > 5*time.Minute { - v.reject(errors.New("service.request.timeout must not exceed 5m")) + if file.Request != nil { + loaded.Request.MaxOperations = v.integer("request.max_operations", file.Request.MaxOperations, 1000) + } else if file.Mode == ModeGateway { + loaded.Request.MaxOperations = 1000 } - request.MaxOperations = v.integer("service.request.max_operations", file.Request.MaxOperations, 1000) - request.MaxReadBytes = v.bytes("service.request.max_read_bytes", file.Request.MaxReadBytes, min(32<<20, grpc.MaxSendMessageBytes/2), grpc.MaxSendMessageBytes/2) - execution := &loaded.Execution - execution.MaxRequests = v.bounded("service.execution.max_requests", file.Execution.MaxRequests, 128, 10000) - execution.MaxBytes = v.bytes("service.execution.max_bytes", file.Execution.MaxBytes, 256<<20, 16<<30) - queue := &execution.Queue - queue.MaxRequests = v.bounded("service.execution.queue.max_requests", file.Execution.Queue.MaxRequests, 1024, 10000) - queue.MaxBytes = v.bytes("service.execution.queue.max_bytes", file.Execution.Queue.MaxBytes, min(32<<20, execution.MaxBytes), 16<<30) - queue.MaxWait = v.duration("service.execution.queue.max_wait", file.Execution.Queue.MaxWait, min(2*time.Second, request.Timeout)) - if queue.MaxWait > request.Timeout { - v.reject(errors.New("service.execution.queue.max_wait cannot exceed service.request.timeout")) + execution := file.Execution + if execution == nil { + execution = &executionFile{} } - scan := &execution.Scan - scan.MaxRequests = v.bounded("service.execution.scan.max_requests", file.Execution.Scan.MaxRequests, max(1, execution.MaxRequests/2), execution.MaxRequests) - scan.MaxBytes = v.bytes("service.execution.scan.max_bytes", file.Execution.Scan.MaxBytes, max(1, execution.MaxBytes/2), execution.MaxBytes) - scan.AdmissionWait = v.duration("service.execution.scan.admission_wait", file.Execution.Scan.AdmissionWait, min(2*time.Second, request.Timeout)) - if scan.AdmissionWait > request.Timeout { - v.reject(errors.New("service.execution.scan.admission_wait cannot exceed service.request.timeout")) - } - loaded.Publish.MaxRequests = v.bounded("service.publish.max_requests", file.Publish.MaxRequests, 32, 10000) - loaded.Publish.MaxBytes = v.bytes("service.publish.max_bytes", file.Publish.MaxBytes, 256<<20, 16<<30) - loaded.Batching = resolveBatching(file.Batching, request, grpc, v) - loaded.Merge.MaxAttempts = v.integer("service.merge.max_attempts", file.Merge.MaxAttempts, 3) + loaded.Execution.MaxSnapshotBytes = v.bytes("execution.max_snapshot_bytes", execution.MaxSnapshotBytes, 32<<20, math.MaxInt) + loaded.Execution.MaxOutputBytes = v.bytes("execution.max_output_bytes", execution.MaxOutputBytes, 32<<20, math.MaxInt) + mongo := &loaded.Execution.MongoDB + mongo.MaxConcurrentWrites = v.integer("execution.mongodb.max_concurrent_writes", execution.MongoDB.MaxConcurrentWrites, 64) + mongo.MaxConcurrentGroups = v.integer("execution.mongodb.max_concurrent_groups", execution.MongoDB.MaxConcurrentGroups, 16) + loaded.Merge.MaxAttempts = v.integer("execution.merge.max_attempts", execution.Merge.MaxAttempts, 3) lua := &loaded.Merge.Lua - lua.Timeout = v.duration("service.merge.lua.timeout", file.Merge.Lua.Timeout, 100*time.Millisecond) - lua.MaxSourceBytes = v.bytes("service.merge.lua.max_source_bytes", file.Merge.Lua.MaxSourceBytes, 64<<10, math.MaxInt) - lua.MaxResultBytes = v.bytes("service.merge.lua.max_result_bytes", file.Merge.Lua.MaxResultBytes, 16<<20, math.MaxInt) - lua.MaxCachedPrograms = v.integer("service.merge.lua.max_cached_programs", file.Merge.Lua.MaxCachedPrograms, 256) - lua.MaxInstructions = v.integer("service.merge.lua.max_instructions", file.Merge.Lua.MaxInstructions, 1_000_000) + lua.Timeout = v.duration("execution.merge.lua.timeout", execution.Merge.Lua.Timeout, 100*time.Millisecond) + lua.MaxSourceBytes = v.bytes("execution.merge.lua.max_source_bytes", execution.Merge.Lua.MaxSourceBytes, 64<<10, math.MaxInt) + lua.MaxResultBytes = v.bytes("execution.merge.lua.max_result_bytes", execution.Merge.Lua.MaxResultBytes, 16<<20, math.MaxInt) + lua.MaxCachedPrograms = v.integer("execution.merge.lua.max_cached_programs", execution.Merge.Lua.MaxCachedPrograms, 256) + lua.MaxInstructions = v.integer("execution.merge.lua.max_instructions", execution.Merge.Lua.MaxInstructions, 1_000_000) + if file.Mode == ModeEngine { + batching := file.Batching + if batching == nil { + batching = &batchingFile{} + } + loaded.Batching = resolveBatching(*batching, grpc, v) + } return loaded } -func resolveBatching(file batchingFile, request *Request, grpc GRPC, v *validator) Batching { +func resolveBatching(file batchingFile, grpc GRPC, v *validator) Batching { var loaded Batching - loaded.MaxWait = v.duration("service.batching.max_wait", file.MaxWait, 2*time.Millisecond) - loaded.MaxOperations = v.integer("service.batching.max_operations", file.MaxOperations, request.MaxOperations) - loaded.MaxBytes = v.bytes("service.batching.max_bytes", file.MaxBytes, 16<<20, math.MaxInt) - loaded.Queue.MaxOperations = v.integer("service.batching.queue.max_operations", file.Queue.MaxOperations, max(10_000, request.MaxOperations)) - loaded.Queue.MaxBytes = v.bytes("service.batching.queue.max_bytes", file.Queue.MaxBytes, max(128<<20, grpc.MaxReceiveMessageBytes), math.MaxInt) - if loaded.MaxOperations > request.MaxOperations { - v.reject(errors.New("service.batching.max_operations cannot exceed service.request.max_operations")) - } - if loaded.Queue.MaxOperations < request.MaxOperations || loaded.Queue.MaxOperations < loaded.MaxOperations { - v.reject(errors.New("service.batching.queue.max_operations must cover one server request and one batch")) + loaded.MaxWait = v.duration("batching.max_wait", file.MaxWait, 2*time.Millisecond) + loaded.MaxOperations = v.integer("batching.max_operations", file.MaxOperations, 1000) + loaded.MaxBytes = v.bytes("batching.max_bytes", file.MaxBytes, 16<<20, math.MaxInt) + loaded.Queue.MaxOperations = v.integer("batching.queue.max_operations", file.Queue.MaxOperations, max(10_000, loaded.MaxOperations)) + loaded.Queue.MaxBytes = v.bytes("batching.queue.max_bytes", file.Queue.MaxBytes, max(128<<20, grpc.MaxReceiveMessageBytes), math.MaxInt) + if loaded.Queue.MaxOperations < loaded.MaxOperations { + v.reject(errors.New("batching.queue.max_operations must cover one batch")) } if loaded.Queue.MaxBytes < grpc.MaxReceiveMessageBytes || loaded.Queue.MaxBytes < loaded.MaxBytes { - v.reject(errors.New("service.batching.queue.max_bytes must cover one gRPC request and one batch")) + v.reject(errors.New("batching.queue.max_bytes must cover one gRPC request and one batch")) } return loaded } diff --git a/internal/config/storage.go b/internal/config/storage.go index b4cee82..90cc57f 100644 --- a/internal/config/storage.go +++ b/internal/config/storage.go @@ -1,18 +1,12 @@ package config import ( - "github.com/liran/sink-go/uri" - "fmt" "strings" ) func resolveStorage(prefix string, file storageFile) (Storage, error) { var loaded Storage - loaded.Name = strings.TrimSpace(file.Name) - if !uri.ValidStore(loaded.Name) { - return loaded, fmt.Errorf("%s.name must be a canonical lowercase Store name", prefix) - } v := validator{} loaded.Driver = Driver(strings.TrimSpace(string(file.Driver))) switch loaded.Driver { @@ -27,8 +21,6 @@ func resolveStorage(prefix string, file storageFile) (Storage, error) { return loaded, fmt.Errorf("%s.mongodb.uri is required when driver is mongodb", prefix) } mongo.MetadataField = valueOrDefault(file.MongoDB.MetadataField, "__sink") - mongo.MaxConcurrentWrites = v.integer(prefix+".mongodb.max_concurrent_writes", file.MongoDB.MaxConcurrentWrites, 64) - mongo.MaxConcurrentGroups = v.integer(prefix+".mongodb.max_concurrent_groups", file.MongoDB.MaxConcurrentGroups, 16) case DriverElasticsearch, DriverOpenSearch: search := &loaded.Search search.Endpoints = nonEmptyValues(file.Search.Endpoints) @@ -57,6 +49,5 @@ func resolveStorage(prefix string, file storageFile) (Storage, error) { default: return loaded, fmt.Errorf("%s.driver must be mongodb, elasticsearch, or opensearch", prefix) } - loaded.Kafka = resolveKafka(prefix+".kafka", file.Kafka, &v) return loaded, v.err } diff --git a/internal/forwarding/budget.go b/internal/forwarding/budget.go index c39f224..04d8b39 100644 --- a/internal/forwarding/budget.go +++ b/internal/forwarding/budget.go @@ -9,7 +9,7 @@ import ( "github.com/liran/sink/internal/storage" ) -const Version = 4 +const Version = 5 // EnvelopeBytes is reserved in the private transport in addition to the public message cap. const EnvelopeBytes = 4096 @@ -38,7 +38,10 @@ func NewTracker(grant *forward.Budget, maximum int) *Tracker { t := &Tracker{} raw := [...]uint64{grant.GetSnapshots(), grant.GetInputs(), grant.GetOutputs(), grant.GetReturns()} for i, size := range raw { - t.limits[i] = int(min(size, uint64(maximum))) + t.limits[i] = int(min(size, uint64(int(^uint(0)>>1)))) + if Kind(i) == Returns { + t.limits[i] = min(t.limits[i], maximum) + } } return t } diff --git a/internal/gateway/batch.go b/internal/gateway/batch.go index 8422987..b7cce32 100644 --- a/internal/gateway/batch.go +++ b/internal/gateway/batch.go @@ -8,7 +8,6 @@ import ( forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -81,7 +80,10 @@ func (s *Server) records(ctx context.Context, req *forward.ForwardRequest) (*for } groups[position].indices = append(groups[position].indices, index) } - remaining := forwarding.FullBudget(s.request.MaxReadBytes) + remaining, err := s.responseBudget(len(addresses)) + if err != nil { + return nil, err + } run := func(group storeGroup, grant *forward.Budget) *forward.Budget { sub := splitRequest(req, group.indices) sub.Grant = grant @@ -112,7 +114,7 @@ func (s *Server) records(ctx context.Context, req *forward.ForwardRequest) (*for for range min(s.config.MaxFanout, len(groups)) { work.Go(func() { for group := range jobs { - run(group, forwarding.FullBudget(s.request.MaxReadBytes)) + run(group, remaining) } }) } diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index f6b3389..ecda440 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -9,7 +9,7 @@ import ( ) func TestRoutesRejectDuplicateStoreNames(t *testing.T) { - loaded, err := config.Decode(strings.NewReader("mode: gateway\ngateway:\n routes:\n - store: a\n target: 127.0.0.1:1\n tls: {insecure: true}\n - store: a\n target: 127.0.0.1:2\n tls: {insecure: true}\n")) + loaded, err := config.Decode(strings.NewReader("mode: gateway\nforwarding:\n routes:\n - store: a\n target: 127.0.0.1:1\n tls: {insecure: true}\n - store: a\n target: 127.0.0.1:2\n tls: {insecure: true}\n"), nil) if err != nil { t.Fatal(err) } diff --git a/internal/gateway/methods.go b/internal/gateway/methods.go index 4397d46..8780678 100644 --- a/internal/gateway/methods.go +++ b/internal/gateway/methods.go @@ -5,7 +5,6 @@ import ( forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -46,7 +45,11 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E return nil, status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Execute{Execute: req} - request := &forward.ForwardRequest{Request: body, Grant: forwarding.FullBudget(s.request.MaxReadBytes)} + grant, err := s.responseBudget(0) + if err != nil { + return nil, err + } + request := &forward.ForwardRequest{Request: body, Grant: grant} ctx, release, err := s.begin(ctx, request) if err != nil { return nil, err @@ -74,7 +77,11 @@ func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.Query return nil, status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Query{Query: req} - request := &forward.ForwardRequest{Request: body, Grant: forwarding.FullBudget(s.request.MaxReadBytes)} + grant, err := s.responseBudget(0) + if err != nil { + return nil, err + } + request := &forward.ForwardRequest{Request: body, Grant: grant} ctx, release, err := s.begin(ctx, request) if err != nil { return nil, err @@ -102,7 +109,11 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count return nil, status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Count{Count: req} - request := &forward.ForwardRequest{Request: body, Grant: forwarding.FullBudget(s.request.MaxReadBytes)} + grant, err := s.responseBudget(0) + if err != nil { + return nil, err + } + request := &forward.ForwardRequest{Request: body, Grant: grant} ctx, release, err := s.begin(ctx, request) if err != nil { return nil, err @@ -130,7 +141,11 @@ func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanRes return nil, status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Scan{Scan: req} - request := &forward.ForwardRequest{Request: body, Grant: forwarding.FullBudget(s.request.MaxReadBytes)} + grant, err := s.responseBudget(0) + if err != nil { + return nil, err + } + request := &forward.ForwardRequest{Request: body, Grant: grant} ctx, release, err := s.begin(ctx, request) if err != nil { return nil, err diff --git a/internal/gateway/response_budget_test.go b/internal/gateway/response_budget_test.go new file mode 100644 index 0000000..45c2d40 --- /dev/null +++ b/internal/gateway/response_budget_test.go @@ -0,0 +1,38 @@ +package gateway + +import ( + "math" + "testing" + + forward "github.com/liran/sink/gen/forward" + "github.com/liran/sink/internal/config" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestResponseBudgetComesFromTransportAndSharesOnlyReturns(t *testing.T) { + request := config.Request{MaxReadBytes: 4096} + server := &Server{request: request} + grant, err := server.responseBudget(2) + if err != nil { + t.Fatal(err) + } + if grant.Returns != 4096-2*1280 { + t.Fatalf("unreserved response envelope: %v", grant) + } + if grant.Snapshots != math.MaxInt64 || grant.Inputs != math.MaxInt64 || grant.Outputs != math.MaxInt64 { + t.Fatalf("transport ceiling leaked into Store-local execution: %v", grant) + } + used := &forward.Budget{Snapshots: 10000, Inputs: 20000, Outputs: 30000, Returns: 100} + consume(grant, used) + if grant.Returns != 4096-2*1280-100 || grant.Snapshots != math.MaxInt64 || grant.Outputs != math.MaxInt64 { + t.Fatalf("settlement charged unrelated Store working sets: %v", grant) + } + if _, err := server.responseBudget(4); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("impossible response must fail before forwarding: %v", err) + } + native, err := server.responseBudget(0) + if err != nil || native.GetReturns() != 4096 { + t.Fatalf("native encoded response: %v %v", native, err) + } +} diff --git a/internal/gateway/server.go b/internal/gateway/server.go index f35bacc..f30ccd1 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -3,6 +3,7 @@ package gateway import ( "context" "errors" + "math" "sync" "sync/atomic" "time" @@ -32,14 +33,20 @@ type Server struct { nativeSequence atomic.Uint64 } type Options struct { - Memory *capacity.Pool - Gateway config.Gateway - Request config.Request - MaxMessageBytes int + Memory *capacity.Pool + Gateway config.Gateway + Request config.Request + MaxMessageBytes int + MaxResponseBytes int } func New(opts Options) (*Server, error) { - if opts.Gateway.MaxRequests <= 0 || opts.Gateway.MaxRequestsPerStore <= 0 || opts.Gateway.MaxBytes <= 0 || opts.Gateway.MaxFanout <= 0 || opts.Gateway.MaxConnections <= 0 || opts.Gateway.IdleTimeout <= 0 || opts.Gateway.DNSRefreshInterval <= 0 || opts.Request.Timeout <= 0 || opts.Request.MaxOperations <= 0 || opts.Request.MaxReadBytes <= 0 || opts.MaxMessageBytes <= 0 { + if opts.MaxResponseBytes == 0 { + opts.MaxResponseBytes = opts.MaxMessageBytes + } + opts.Request.MaxReadBytes = opts.MaxResponseBytes + + if opts.Gateway.MaxRequests <= 0 || opts.Gateway.MaxRequestsPerStore <= 0 || opts.Gateway.MaxBytes <= 0 || opts.Gateway.MaxFanout <= 0 || opts.Gateway.MaxConnections <= 0 || opts.Gateway.IdleTimeout <= 0 || opts.Gateway.DNSRefreshInterval <= 0 || opts.Request.MaxOperations <= 0 || opts.MaxMessageBytes <= 0 || opts.MaxResponseBytes <= 0 { return nil, errors.New("gateway limits must be positive") } initial, err := newSnapshot(opts.Gateway.Routes) @@ -96,7 +103,7 @@ func (s *Server) begin(ctx context.Context, req *forward.ForwardRequest) (contex s.metrics.inFlight.Inc() s.metrics.bytes.Add(float64(charge)) s.mu.Unlock() - ctx, cancel := context.WithTimeout(ctx, s.request.Timeout) + ctx, cancel := context.WithCancel(ctx) release := func() { cancel() s.mu.Lock() @@ -163,9 +170,6 @@ func validUsage(grant, used *forward.Budget) bool { return used.GetSnapshots() <= grant.GetSnapshots() && used.GetInputs() <= grant.GetInputs() && used.GetOutputs() <= grant.GetOutputs() && used.GetReturns() <= grant.GetReturns() } func consume(remaining, used *forward.Budget) { - remaining.Snapshots -= used.GetSnapshots() - remaining.Inputs -= used.GetInputs() - remaining.Outputs -= used.GetOutputs() remaining.Returns -= used.GetReturns() } func routeFor(view *snapshot, store string) (Route, error) { @@ -203,7 +207,7 @@ func localRejection(route Route, code codes.Code, message string) *forward.Forwa } func (s *Server) beginMemory(ctx context.Context, req *forward.ForwardRequest) (context.Context, func(), error) { - ctx, cancel := context.WithTimeout(ctx, s.request.Timeout) + ctx, cancel := context.WithCancel(ctx) scope := capacity.FromContext(ctx) release := func() {} if scope == nil { @@ -220,3 +224,22 @@ func (s *Server) beginMemory(ctx context.Context, req *forward.ForwardRequest) ( done := func() { cancel(); release(); s.metrics.inFlight.Dec() } return ctx, done, nil } + +// responseBudget derives public payload capacity from the transport ceiling. +// Result envelopes, revision tokens and bounded failures are reserved before +// forwarding; Store-local snapshots and candidates have independent limits. +func (s *Server) responseBudget(operations int) (*forward.Budget, error) { + overhead := 0 + if operations > 0 { + if operations > s.request.MaxReadBytes/1280 { + return nil, status.Error(codes.ResourceExhausted, "result envelopes exceed gRPC send limit") + } + overhead = operations * 1280 + } + bytes := s.request.MaxReadBytes - overhead + if bytes <= 0 { + return nil, status.Error(codes.ResourceExhausted, "response budget is exhausted") + } + budget := &forward.Budget{Snapshots: math.MaxInt64, Inputs: math.MaxInt64, Outputs: math.MaxInt64, Returns: uint64(bytes)} + return budget, nil +} diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go index 3505434..6aa44bd 100644 --- a/internal/logging/logging_test.go +++ b/internal/logging/logging_test.go @@ -29,7 +29,7 @@ func (b *lockedBuffer) String() string { b.mu.Lock(); defer b.mu.Unlock(); retur func testConfig(t *testing.T) config.Logging { t.Helper() - loaded, err := config.Decode(strings.NewReader("mode: engine\nstorage:\n name: primary\n driver: mongodb\n mongodb:\n uri: mongodb://localhost:27017\n")) + loaded, err := config.Decode(strings.NewReader("mode: engine\n"), strings.NewReader("name: primary\nstorage:\n driver: mongodb\n mongodb:\n uri: mongodb://localhost:27017\n")) if err != nil { t.Fatal(err) } diff --git a/internal/protocol/memory.go b/internal/protocol/memory.go index 07db141..2b83355 100644 --- a/internal/protocol/memory.go +++ b/internal/protocol/memory.go @@ -4,7 +4,6 @@ import ( "context" "errors" "strings" - "time" forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" @@ -18,18 +17,11 @@ import ( // MemoryStats owns the RPC reference until gRPC finishes it. Batch producers // and encoded transport buffers keep independent references past cancellation. type MemoryStats struct { - Pool *capacity.Pool - Timeout time.Duration + Pool *capacity.Pool } -type memoryCancelKey struct{} - func (h *MemoryStats) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { if strings.HasPrefix(info.FullMethodName, "/sink.") { - if h.Timeout > 0 { - timed, cancel := context.WithTimeout(ctx, h.Timeout) - ctx = context.WithValue(timed, memoryCancelKey{}, cancel) - } return capacity.WithScope(ctx, h.Pool.NewScope()) } return ctx @@ -38,9 +30,7 @@ func (h *MemoryStats) TagRPC(ctx context.Context, info *stats.RPCTagInfo) contex func (*MemoryStats) HandleRPC(ctx context.Context, event stats.RPCStats) { if _, ok := event.(*stats.End); ok { capacity.FromContext(ctx).Release() - if cancel, ok := ctx.Value(memoryCancelKey{}).(context.CancelFunc); ok { - cancel() - } + } } func (*MemoryStats) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { return ctx } diff --git a/internal/service/admission.go b/internal/service/admission.go index 2f27a18..fa531cb 100644 --- a/internal/service/admission.go +++ b/internal/service/admission.go @@ -16,7 +16,13 @@ import ( "google.golang.org/grpc/status" ) -const defaultRequestTimeout = 30 * time.Second +// A zero timeout preserves the caller's deadline without imposing a server limit. +func executionContext(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { + if timeout == 0 { + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, timeout) +} // Publishing has its own bounded pool so storage latency and snapshot waiters // cannot consume the capacity needed to durably enqueue asynchronous work. @@ -198,10 +204,10 @@ func (s *admissionPool) admitRequest(ctx context.Context, request admissionReque if timeout == 0 { timeout = s.requestTimeout } - if request.direct { + if request.direct && timeout > 0 { timeout -= time.Since(started) } - execution, cancel := context.WithTimeout(ctx, timeout) + execution, cancel := executionContext(ctx, timeout) release := func() { cancel() s.admissionMu.Lock() @@ -435,7 +441,7 @@ func returningCallerCount(req *sink.WriteRequest, budgets *requestBudgets) int { // The application always supplies the process pool. The old admission fields // remain for explicit legacy-limit regression and benchmark fixtures. func (s *Server) admitMemory(ctx context.Context, request admissionRequest) (context.Context, context.CancelFunc, error) { - execution, cancel := context.WithTimeout(ctx, s.requestTimeout) + execution, cancel := executionContext(ctx, s.requestTimeout) scope := capacity.FromContext(execution) releaseScope := func() {} if scope == nil { diff --git a/internal/service/batcher.go b/internal/service/batcher.go index e12be50..1adbf26 100644 --- a/internal/service/batcher.go +++ b/internal/service/batcher.go @@ -101,9 +101,7 @@ func newRequestBatcher[Request any, Response any](opts requestBatcherOptions[Req if opts.Unlimited { batcher.maxConcurrent = 0 } - if batcher.executionTimeout == 0 { - batcher.executionTimeout = defaultRequestTimeout - } + batcher.waitGroup.Add(1) go batcher.run() return batcher @@ -433,23 +431,36 @@ func batchExecutionContext[Request any, Response any]( calls []*batchCall[Request, Response], timeout time.Duration, ) (context.Context, context.CancelFunc) { - // Keep shared work alive while at least one caller still needs it, but - // never extend execution beyond the server's own time budget. - limit := time.Now().Add(timeout) + // Shared work survives until every caller cancels. A caller without a + // deadline must not inherit another caller's deadline or a synthetic limit. var latest time.Time + unbounded := false for _, call := range calls { deadline, ok := call.ctx.Deadline() if !ok { - deadline = limit + unbounded = true } if deadline.After(latest) { latest = deadline } } - if latest.IsZero() || latest.After(limit) { - latest = limit + if unbounded { + latest = time.Time{} + } + if timeout > 0 { + limit := time.Now().Add(timeout) + if latest.IsZero() || latest.After(limit) { + latest = limit + } + } + var ctx context.Context + var cancel context.CancelFunc + if latest.IsZero() { + ctx, cancel = context.WithCancel(parent) + } else { + ctx, cancel = context.WithDeadline(parent, latest) } - ctx, cancel := context.WithDeadline(parent, latest) + var mu sync.Mutex remaining := len(calls) stops := make([]func() bool, 0, len(calls)) diff --git a/internal/service/batching_regressions_test.go b/internal/service/batching_regressions_test.go index 37e4149..ac3c5d2 100644 --- a/internal/service/batching_regressions_test.go +++ b/internal/service/batching_regressions_test.go @@ -53,6 +53,10 @@ func TestRegressionMicrobatchMergeBudgetIsolation(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 makeCall := func(key string) *batchCall[*sink.WriteRequest, *sink.WriteResponse] { op := completionMerge(key, 1) op.GetMerge().LuaProgram.Source = []byte(`return function(current, incoming) return incoming end`) @@ -84,6 +88,10 @@ func TestRegressionMicrobatchReadBudgetIsolation(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 var calls []*batchCall[*sink.ReadRequest, *sink.ReadResponse] for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -119,6 +127,10 @@ func TestMicrobatchReadKeepsOversizedCallerIsolatedForSharedKey(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 var operations []*sink.ReadOperation for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -156,6 +168,10 @@ func TestMicrobatchMergeBudgetFailureDoesNotLeakIntoNextCaller(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 150 + + server.server.maxSnapshotBytes = 150 + + server.server.maxOutputBytes = 150 if present { for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -205,6 +221,10 @@ func TestMicrobatchBudgetsSplitWithinExecutionMemoryLimit(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 server.server.maxInFlightBytes = 1000 + failureResponseBytes(1) switch method { case "Write": @@ -371,6 +391,10 @@ func TestMicrobatchConditionalWritesKeepEachCallersInputAndOutputBudget(t *testi backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -407,6 +431,10 @@ func TestCancelledRPCIsOmittedFromLaterMemoryLimitedSegment(t *testing.T) { backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 8), blocked: "a", release: make(chan struct{})} server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 server.server.maxInFlightBytes = 1000 + failureResponseBytes(1) cancelled, cancel := context.WithCancel(t.Context()) defer cancel() @@ -455,6 +483,10 @@ func TestHotRecordSharesPhysicalReservationAcrossRPCBudgets(t *testing.T) { backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 8)} server := completionServer(t, backend) server.server.maxReadBytes = 256 + + server.server.maxSnapshotBytes = 256 + + server.server.maxOutputBytes = 256 server.server.maxInFlightBytes = 2048 + failureResponseBytes(4) var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] for range 4 { diff --git a/internal/service/batching_server.go b/internal/service/batching_server.go index b526291..589048b 100644 --- a/internal/service/batching_server.go +++ b/internal/service/batching_server.go @@ -124,13 +124,13 @@ func normalizeBatchingOptions(server *Server, opts BatchingOptions) (BatchingOpt opts.MaxWait = defaultBatchMaxWait } if opts.MaxOperations == 0 { - opts.MaxOperations = server.maxOperations + opts.MaxOperations = defaultMaxOperations } if opts.MaxBytes == 0 { opts.MaxBytes = defaultBatchMaxBytes } if opts.MaxQueuedOperations == 0 { - opts.MaxQueuedOperations = max(defaultBatchMaxQueuedOperations, server.maxOperations) + opts.MaxQueuedOperations = max(defaultBatchMaxQueuedOperations, opts.MaxOperations) } if opts.MaxQueuedBytes == 0 { opts.MaxQueuedBytes = max(defaultBatchMaxQueuedBytes, opts.MaxBytes) @@ -139,7 +139,7 @@ func normalizeBatchingOptions(server *Server, opts BatchingOptions) (BatchingOpt var empty BatchingOptions return empty, fmt.Errorf("create synchronous batching server: max operations %d exceeds server limit %d", opts.MaxOperations, server.maxOperations) } - if opts.MaxQueuedOperations < server.maxOperations || opts.MaxQueuedOperations < opts.MaxOperations { + if opts.MaxQueuedOperations < opts.MaxOperations { var empty BatchingOptions return empty, errors.New("create synchronous batching server: queued operation limit must cover one server request and one batch") } @@ -163,7 +163,7 @@ func (s *BatchingServer) Read(ctx context.Context, req *sink.ReadRequest) (*sink if err := protocol.CheckStore(req, s.server.boundStore); err != nil { return nil, err } - ctx, cancel := context.WithTimeout(ctx, s.server.requestTimeout) + ctx, cancel := executionContext(ctx, s.server.requestTimeout) defer cancel() if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "read request must contain operations") @@ -183,7 +183,7 @@ func (s *BatchingServer) Write(ctx context.Context, req *sink.WriteRequest) (*si if err := protocol.CheckStore(req, s.server.boundStore); err != nil { return nil, err } - ctx, cancel := context.WithTimeout(ctx, s.server.requestTimeout) + ctx, cancel := executionContext(ctx, s.server.requestTimeout) defer cancel() if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "write request must contain operations") @@ -256,7 +256,7 @@ func (s *BatchingServer) Delete(ctx context.Context, req *sink.DeleteRequest) (* if err := protocol.CheckStore(req, s.server.boundStore); err != nil { return nil, err } - ctx, cancel := context.WithTimeout(ctx, s.server.requestTimeout) + ctx, cancel := executionContext(ctx, s.server.requestTimeout) defer cancel() if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "delete request must contain operations") diff --git a/internal/service/caller_deadline_test.go b/internal/service/caller_deadline_test.go new file mode 100644 index 0000000..e75643a --- /dev/null +++ b/internal/service/caller_deadline_test.go @@ -0,0 +1,45 @@ +package service + +import ( + "context" + "testing" + "time" +) + +func TestBatchWithoutCallerDeadlineHasNoSyntheticDeadline(t *testing.T) { + short, cancelShort := context.WithTimeout(context.Background(), time.Hour) + defer cancelShort() + unbounded, cancelUnbounded := context.WithCancel(context.Background()) + defer cancelUnbounded() + calls := []*batchCall[int, int]{{ctx: short}, {ctx: unbounded}} + ctx, release := batchExecutionContext(context.Background(), calls, 0) + defer release() + if deadline, ok := ctx.Deadline(); ok { + t.Fatalf("invented deadline: %v", deadline) + } + cancelShort() + if ctx.Err() != nil { + t.Fatal("one canceled caller stopped shared work") + } + cancelUnbounded() + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("all callers canceled but shared work survived") + } +} + +func TestBatchInheritsLatestCallerDeadline(t *testing.T) { + first, cancelFirst := context.WithTimeout(context.Background(), time.Minute) + defer cancelFirst() + last, cancelLast := context.WithTimeout(context.Background(), time.Hour) + defer cancelLast() + calls := []*batchCall[int, int]{{ctx: first}, {ctx: last}} + ctx, cancel := batchExecutionContext(context.Background(), calls, 0) + defer cancel() + want, _ := last.Deadline() + got, ok := ctx.Deadline() + if !ok || !got.Equal(want) { + t.Fatalf("deadline = %v, want %v", got, want) + } +} diff --git a/internal/service/native.go b/internal/service/native.go index d1eb2b3..7b57b7a 100644 --- a/internal/service/native.go +++ b/internal/service/native.go @@ -92,7 +92,7 @@ func nativeStatus(err error) error { } func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.ExecuteResponse, error) { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Outputs, s.maxReadBytes) + maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) if maximum <= 0 { return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") } @@ -147,16 +147,15 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E } func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanResponse, error) { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Outputs, s.maxReadBytes) + maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) if maximum <= 0 { return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") } if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } - // Admission and backend execution share one page deadline, even when the - // caller did not provide one. Admission has an additional, shorter bound. - ctx, cancel := context.WithTimeout(ctx, s.requestTimeout) + // Preserve the caller deadline across admission and backend execution. + ctx, cancel := executionContext(ctx, s.requestTimeout) defer cancel() maximum = min(maximum, 4<<20) request, err := nativeRequest(req.GetCommand(), maximum) diff --git a/internal/service/options.go b/internal/service/options.go index 7ef9563..b128ad9 100644 --- a/internal/service/options.go +++ b/internal/service/options.go @@ -35,6 +35,8 @@ type Options struct { MaxPublishRequests int MaxPublishBytes int MaxReadBytes int + MaxSnapshotBytes int + MaxOutputBytes int MaxScanRequests int MaxScanBytes int ScanAdmissionWait time.Duration @@ -59,18 +61,25 @@ func New(opts Options) (*Server, error) { if opts.RequestTimeout < 0 || opts.MaxInFlightRequests < 0 || opts.MaxInFlightBytes < 0 || opts.MaxReadBytes < 0 { return nil, errors.New("create Sink server: resource limits cannot be negative") } - if opts.RequestTimeout == 0 { - opts.RequestTimeout = defaultRequestTimeout - } + if opts.MaxInFlightRequests == 0 { opts.MaxInFlightRequests = 128 } if opts.MaxInFlightBytes == 0 { opts.MaxInFlightBytes = 256 << 20 } + if opts.MaxSnapshotBytes < 0 || opts.MaxOutputBytes < 0 { + return nil, errors.New("execution budgets cannot be negative") + } if opts.MaxReadBytes == 0 { opts.MaxReadBytes = storage.DefaultMaxReadBytes } + if opts.MaxSnapshotBytes == 0 { + opts.MaxSnapshotBytes = opts.MaxReadBytes + } + if opts.MaxOutputBytes == 0 { + opts.MaxOutputBytes = opts.MaxReadBytes + } if opts.MaxAdmissionRequests < 0 || opts.MaxAdmissionBytes < 0 || opts.AdmissionWait < 0 { return nil, errors.New("create Sink server: admission queue limits cannot be negative") } @@ -83,7 +92,9 @@ func New(opts Options) (*Server, error) { if opts.AdmissionWait == 0 { opts.AdmissionWait = 2 * time.Second } - opts.AdmissionWait = min(opts.AdmissionWait, opts.RequestTimeout) + if opts.RequestTimeout > 0 { + opts.AdmissionWait = min(opts.AdmissionWait, opts.RequestTimeout) + } if opts.MaxPublishRequests < 0 || opts.MaxPublishBytes < 0 { return nil, errors.New("create Sink server: publish limits cannot be negative") } @@ -105,14 +116,16 @@ func New(opts Options) (*Server, error) { if opts.ScanAdmissionWait == 0 { opts.ScanAdmissionWait = 2 * time.Second } - opts.ScanAdmissionWait = min(opts.ScanAdmissionWait, opts.RequestTimeout) + if opts.RequestTimeout > 0 { + opts.ScanAdmissionWait = min(opts.ScanAdmissionWait, opts.RequestTimeout) + } if opts.MaxScanRequests > opts.MaxInFlightRequests || opts.MaxScanBytes > opts.MaxInFlightBytes { return nil, errors.New("create Sink server: scan limits cannot exceed total limits") } maxOperations := opts.MaxOperations if maxOperations == 0 { - maxOperations = defaultMaxOperations + maxOperations = int(^uint(0) >> 1) } maxMergeAttempts := opts.MaxMergeAttempts if maxMergeAttempts == 0 { @@ -149,6 +162,8 @@ func New(opts Options) (*Server, error) { maxMergeAttempts: maxMergeAttempts, metrics: opts.Metrics, maxReadBytes: opts.MaxReadBytes, + maxSnapshotBytes: opts.MaxSnapshotBytes, + maxOutputBytes: opts.MaxOutputBytes, admissionPool: executionAdmission, publishAdmission: publishAdmission, } diff --git a/internal/service/query.go b/internal/service/query.go index 7689866..04a62e1 100644 --- a/internal/service/query.go +++ b/internal/service/query.go @@ -14,7 +14,7 @@ import ( const maxCountResponseBytes = 256 << 10 func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.QueryResponse, error) { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Outputs, s.maxReadBytes) + maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) if maximum <= 0 { return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") } @@ -75,7 +75,7 @@ func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.Query } func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.CountResponse, error) { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Outputs, s.maxReadBytes) + maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) if maximum <= 0 { return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") } diff --git a/internal/service/read.go b/internal/service/read.go index 27c57e4..c277d12 100644 --- a/internal/service/read.go +++ b/internal/service/read.go @@ -124,7 +124,7 @@ func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *reque // Charge copies in each original RPC's order, including repeated keys. // Check the aggregate output size before allocating any document copies. - outputBudgets := budgets.fresh(forwarding.Outputs, s.maxReadBytes) + outputBudgets := budgets.fresh(forwarding.Returns, s.maxReadBytes) outputBytes := make([]int, budgets.callerCount()) for index, operationIndex := range operationIndexes { owner := budgets.owner(operationIndex) diff --git a/internal/service/server.go b/internal/service/server.go index ae71064..8cb90e4 100644 --- a/internal/service/server.go +++ b/internal/service/server.go @@ -33,6 +33,8 @@ type Server struct { maxMergeAttempts int metrics *sinkmetrics.Metrics maxReadBytes int + maxSnapshotBytes int + maxOutputBytes int publishAdmission *admissionPool } diff --git a/internal/service/server_test.go b/internal/service/server_test.go index 4144cef..520df7e 100644 --- a/internal/service/server_test.go +++ b/internal/service/server_test.go @@ -640,7 +640,8 @@ func newTestServer(t testing.TB, store storage.Storage, publisher queue.Publishe Publisher: publisher, MaxMergeAttempts: testMergeAttempts, // Small fixture documents allow many independent RPC budgets in one batch. - MaxReadBytes: 1 << 20, + MaxReadBytes: 1 << 20, + MaxOperations: 1000, } server, err := service.New(options) if err != nil { diff --git a/internal/service/sync_capacity_test.go b/internal/service/sync_capacity_test.go index 7573677..2c6ba49 100644 --- a/internal/service/sync_capacity_test.go +++ b/internal/service/sync_capacity_test.go @@ -65,6 +65,10 @@ func TestSynchronousMergeStreamsLargeSnapshotsAndOutputs(t *testing.T) { } server := completionServer(t, backend) server.server.maxReadBytes = 1024 + + server.server.maxSnapshotBytes = 1024 + + server.server.maxOutputBytes = 1024 var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] for index := range 8 { key := fmt.Sprintf("record-%d", index) @@ -117,6 +121,10 @@ func TestSynchronousChunksKeepCallerQuotaAndSharedRecordIsolation(t *testing.T) backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 400 + + server.server.maxSnapshotBytes = 400 + + server.server.maxOutputBytes = 400 for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) if err != nil { diff --git a/internal/service/write.go b/internal/service/write.go index 9d96165..94b03de 100644 --- a/internal/service/write.go +++ b/internal/service/write.go @@ -372,10 +372,10 @@ func (s *Server) executeWriteAttempt( attempt := writeAttempt{ options: opts, results: results, - snapshots: opts.budgets.fresh(forwarding.Snapshots, s.maxReadBytes), + snapshots: opts.budgets.fresh(forwarding.Snapshots, s.maxSnapshotBytes), inputs: opts.budgets.fresh(forwarding.Inputs, s.maxReadBytes), - outputs: opts.budgets.fresh(forwarding.Outputs, s.maxReadBytes), - output: storage.NewReadBudget(s.maxReadBytes), + outputs: opts.budgets.fresh(forwarding.Outputs, s.maxOutputBytes), + output: storage.NewReadBudget(s.maxOutputBytes), } next := make([]writeGroup, 0) pending := groups @@ -440,7 +440,7 @@ type writeChunkOutcome struct { } func (s *Server) readWriteSnapshots(ctx context.Context, groups []writeGroup, attempt *writeAttempt) (storage.ReadResponse, error) { - working := storage.NewReadBudget(s.maxReadBytes) + working := storage.NewReadBudget(s.maxSnapshotBytes) readOperations := make([]storage.ReadOperation, 0, len(groups)) for _, group := range groups { operation := group.operations[0] @@ -456,7 +456,7 @@ func (s *Server) readWriteSnapshots(ctx context.Context, groups []writeGroup, at readOperation := storage.ReadOperation{Address: operation.address, Budget: budget} readOperations = append(readOperations, readOperation) } - readRequest := storage.ReadRequest{Operations: readOperations, Budget: storage.NewReadBudget(s.maxReadBytes)} + readRequest := storage.ReadRequest{Operations: readOperations, Budget: storage.NewReadBudget(s.maxSnapshotBytes)} started := time.Now() readResponse, err := s.storage.Read(ctx, readRequest) attempt.options.observation.phase("storage_read", started) @@ -519,7 +519,7 @@ func (s *Server) applyWriteSnapshots(ctx context.Context, groups []writeGroup, s continue } } - if opts.budgets.callerCount() > 1 && len(candidates) > 0 && candidateBytes > s.maxReadBytes-outputBytes { + if opts.budgets.callerCount() > 1 && len(candidates) > 0 && candidateBytes > s.maxOutputBytes-outputBytes { attempt.pendingCandidateBytes = candidateBytes conflicts, err := s.commitWriteCandidates(ctx, candidates, attempt) attempt.pendingCandidateBytes = 0 From 704fd4583ae33ee3157b9296e7bf5a61958a987b Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 04:37:21 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=93=9D=20docs:=20migrate=20role=20con?= =?UTF-8?q?figurations=20and=20deployment=20examples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 +- configs/README.md | 14 +- configs/engine.yaml | 142 ++++--------- configs/gateway.yaml | 72 +++---- configs/stores/primary.yaml | 28 +++ configs/worker.yaml | 129 +++--------- docs/batching.md | 6 +- docs/configuration-migration.md | 51 +++-- docs/configuration.md | 230 ++++++++++----------- docs/design/demand-based-admission.md | 7 +- docs/design/store-isolated-architecture.md | 2 +- docs/design/store-isolated-validation.md | 2 +- docs/document-write-flow.md | 4 +- docs/lua-testing.md | 6 +- docs/merge-folding.md | 4 +- docs/native-access.md | 12 +- docs/observability.md | 2 +- docs/record-addresses.md | 2 +- docs/reliability.md | 6 +- docs/rolling-upgrades.md | 6 +- docs/store-isolation.md | 36 ++-- examples/kubernetes/engine-deployment.yaml | 2 +- examples/kubernetes/sink.yaml | 58 ++---- examples/kubernetes/store.yaml | 6 + examples/quickstart/README.md | 5 +- examples/quickstart/compose.yaml | 6 +- examples/quickstart/engine.yaml | 73 +++---- examples/quickstart/gateway.yaml | 35 ++-- examples/quickstart/store.yaml | 28 +++ examples/quickstart/worker.yaml | 75 +++---- 30 files changed, 451 insertions(+), 609 deletions(-) create mode 100644 configs/stores/primary.yaml create mode 100644 examples/kubernetes/store.yaml create mode 100644 examples/quickstart/store.yaml diff --git a/README.md b/README.md index 9db4ba5..f8b7490 100644 --- a/README.md +++ b/README.md @@ -154,21 +154,24 @@ Choose the annotated configuration for the component you are deploying: | Engine | [configs/engine.yaml](configs/engine.yaml) | One Store's synchronous execution and optional asynchronous publication. | | Worker | [configs/worker.yaml](configs/worker.yaml) | One Store's Kafka consumption and mutation execution. | -For an Engine, copy its example, edit the backend connection, and mount the file: +For an Engine, copy the role and shared Store examples, edit the Store connection, +and mount both files. Clients connect through Gateway. ```shell cp configs/engine.yaml config.yaml -# Edit config.yaml for the target backend. +cp configs/stores/primary.yaml store.yaml +# Edit store.yaml for the target backend. docker run --rm -p 8080:8080 \ --mount type=bind,source="$(pwd)/config.yaml",target=/etc/sink/config.yaml,readonly \ - ghcr.io/batchstream/sink:latest --config /etc/sink/config.yaml + --mount type=bind,source="$(pwd)/store.yaml",target=/etc/sink/store.yaml,readonly \ + ghcr.io/batchstream/sink:latest --config /etc/sink/config.yaml --store-config /etc/sink/store.yaml ``` HTTP health endpoints are always available at `health.address` (default `:8081`). Prometheus uses a separate `prometheus.address` (default `:9090`) and requires `prometheus.enabled: true`. Publish the corresponding ports when needed. -Gateway includes its Store routes under `gateway.routes`. Restart after changing +Gateway includes its Store routes under `forwarding.routes`. Restart after changing configuration; routes are loaded only at startup. Worker needs its own configuration and no public gRPC port. The [quickstart](examples/quickstart/README.md) runs all three roles. diff --git a/configs/README.md b/configs/README.md index eca292b..5cf81f4 100644 --- a/configs/README.md +++ b/configs/README.md @@ -1,8 +1,7 @@ # Component configuration examples Copy the file for the component you are deploying. Every field has a comment; -MongoDB is the active backend example, with a search-driver alternative in the -Engine and Worker files. +MongoDB is the active backend example in the shared Store file. Both backend examples include file-based credential alternatives (`uri_file`, `username_file`, `password_file`, and `api_key_file`). Remove the matching inline @@ -13,8 +12,9 @@ API key. Credential files are read at startup; restart after rotating them. | File | Component | | --- | --- | | [gateway.yaml](gateway.yaml) | Public request limits, inline Store routes, and forwarding capacity. | -| [engine.yaml](engine.yaml) | One Store's database, execution, batching, Lua, and optional Kafka publishing. | -| [worker.yaml](worker.yaml) | One Store's database, Kafka consumption, execution, and Lua. | +| [engine.yaml](engine.yaml) | Execution, batching, Lua, and Kafka publishing buffer. | +| [worker.yaml](worker.yaml) | Kafka consumption, execution, and Lua. | +| [stores/primary.yaml](stores/primary.yaml) | Shared Store identity, database connection and Kafka Topic policy. | Every role serves `/livez` and `/readyz` on `health.address` (default `:8081`). Prometheus uses its own `prometheus.address` (default `:9090`) and is disabled @@ -23,15 +23,15 @@ unless `prometheus.enabled: true`. Disabling metrics never disables health check Logs default to warn-level text on stderr. See the [logging reference](../docs/logging.md) for JSON output, direct OTLP export, log levels and optional severe-failure bodies. -Gateway routes live under `gateway.routes` and all configured routes are active. +Gateway routes live under `forwarding.routes` and all configured routes are active. Configuration is loaded at startup; restart the component after changes. Validate without connecting to databases or Kafka, from the repository root: ```sh sink config check --config configs/gateway.yaml -sink config check --config configs/engine.yaml -sink config check --config configs/worker.yaml +sink config check --config configs/engine.yaml --store-config configs/stores/primary.yaml +sink config check --config configs/worker.yaml --store-config configs/stores/primary.yaml ``` See the [configuration reference](../docs/configuration.md) for constraints and diff --git a/configs/engine.yaml b/configs/engine.yaml index ba2278b..b7e715c 100644 --- a/configs/engine.yaml +++ b/configs/engine.yaml @@ -1,115 +1,47 @@ -# Engine: synchronous RPC execution and optional Kafka publication for one Store. -# Values are defaults unless marked as examples or derived defaults. -# Capacities are per process, not cluster-wide or hard RSS limits. +# Engine runtime configuration. Unknown and wrong-role fields are rejected. # Byte sizes accept KiB/MiB/GiB or integer bytes; durations accept ms/s/m/h. -# Positive limits cannot be disabled with 0. Unknown fields are rejected. -# Validate offline: sink config check --config configs/engine.yaml -# See ../docs/configuration.md for constraints and supported storage drivers. -mode: engine # Required process role. +# Validate: sink config check --config configs/engine.yaml --store-config configs/stores/primary.yaml +mode: engine -# One shared process pool; max_bytes is normally detected automatically. -memory: - # max_bytes: 256MiB # Optional explicit capacity, including completion reserve. - burst_percent: 10 # Completion reserve; 1..99; new requests use ordinary capacity. - wait_timeout: 2s # Response growth wait, bounded by the original request deadline. - -# Public RPC and gRPC health listener. grpc: - address: ":8080" # TCP listen address. - max_receive_message_bytes: 64MiB # Maximum encoded inbound gRPC message. - max_send_message_bytes: 64MiB # Maximum encoded outbound gRPC message. - -# Dedicated health listener; always available independently of Prometheus. -health: - address: ":8081" # Listen address for /livez and /readyz; default :8081. - -# Separate optional Prometheus listener. -prometheus: - enabled: false # Start /metrics on its own port; does not affect health; default false. - address: ":9090" # Metrics-only listen address, used only when enabled; default :9090. - -# Bind this process to one Store. Its replicas must use the same Store name. -storage: - name: primary # Required globally unique Store name; shared by all its replicas. - driver: mongodb # Required: mongodb, elasticsearch, or opensearch. - - # Credential-file alternatives below are optional; remove the matching inline key, - # even if empty, before enabling its _file variant. Paths must be absolute. - # Files contain 1 byte..1MiB of UTF-8 without NUL; spaces/newlines are used exactly. - # Files are read at startup and by config check; restart after credential rotation. - - # MongoDB driver settings. Replace with the search block for a search driver. - mongodb: - uri: mongodb://mongodb:27017 # Example; complete connection URI; required unless uri_file is set. - # uri_file: /etc/sink-secrets/mongodb-uri # Example mounted URI file; omit uri when enabled. - metadata_field: __sink # Internal revision field; hidden from clients; default __sink. - # metadata_field cannot be _id or contain a dot, dollar sign, or NUL. - max_concurrent_writes: 64 # Concurrent conditional writes per process; >0; default 64. - max_concurrent_groups: 16 # Concurrent collection groups shared by all calls; >0; default 16. - - # Search driver alternative: remove mongodb, set driver to elasticsearch or opensearch, - # and uncomment the search block. Leave nested file alternatives commented unless used. - # Basic auth requires both username and password; cannot be combined with an API key. - # search: # Backend endpoints and authentication for this Store only. - # endpoints: [http://opensearch:9200] # Example HTTP(S) URLs; at least one required; no default. - # username: "" # Optional basic-auth user; default empty; pair with password/password_file. - # # username_file: /etc/sink-secrets/search-username # Example user file; omit username. - # password: "" # Optional basic-auth password; default empty; pair with username/username_file. - # # password_file: /etc/sink-secrets/search-password # Example password file; omit password. - # api_key: "" # Optional API key; default empty; cannot be combined with basic auth. - # # api_key_file: /etc/sink-secrets/search-api-key # Example API-key file; omit api_key. - - # Optional asynchronous publication; enable it when deploying a Worker. - kafka: - enabled: false # Set true to accept asynchronous mutations; default false. - brokers: [kafka:9092] # Example bootstrap addresses; required when enabled. - # Source topic policy. Matching Engine/Worker replicas must use the same policy. - topic: - name: primary-mutations # Example; required when Kafka is enabled. - partitions: 4 # Exact source/DLQ partition count; changes need a drained migration. - replication_factor: 2 # Replicas per partition; requires enough Kafka brokers. - retention: 72h # Source topic retention; must be at least 1ms. - min_insync_replicas: 2 # Default min(2, replication_factor); cannot exceed it. - max_record_bytes: 900KiB # Encoded mutation plus key; <= producer buffer, <=64MiB. - # Buffer capacity for asynchronous mutation publication. - producer: - max_buffered_bytes: 64MiB # Additional to service budgets; maximum 1GiB. - # Engine reconciles DLQ policy; Worker delivers permanent failures there. - dead_letter: - topic: primary-mutations.dlq # Default .dlq; must differ from source. - retention: 720h # Independent DLQ retention (30 days); at least 1ms. - -# Request handling, synchronous execution, asynchronous publishing, and Lua. -service: - # Per-original-request limits, including requests split/coalesced by Gateway/Engine. - request: - timeout: 30s # Overall unary/page deadline including waits; <=300s; caller may shorten. - max_read_bytes: 32MiB # Read/returned-write/native document budget; <=half gRPC send limit. - max_operations: 1000 # Maximum operations in one Read, Write, or Delete request. - - # Cross-request batching; inputs also acquire the shared memory pool. - batching: - max_wait: 2ms # Collection delay starting with the first request in a batch. - max_operations: 1000 # Default request.max_operations; cannot exceed that limit. - max_bytes: 16MiB # Encoded batch target; a larger valid single RPC can run alone. - # Separate bounded queue for each batched method: Read, Write, and Delete. - queue: - max_operations: 10000 # Default max(10000, request.max_operations); >= batch target. - max_bytes: 128MiB # Default max(128MiB, grpc receive limit); >= receive/batch limits. - - # Conditional mutation conflict handling and Lua limits, shared by Engine/Worker. + address: ":8080" + max_receive_message_bytes: 64MiB + max_send_message_bytes: 64MiB +memory: + # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. + burst_percent: 10 # Completion reserve, 1..99. + wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. +batching: + max_wait: 2ms # Collection delay, not an execution timeout. + max_operations: 1000 + max_bytes: 16MiB + queue: + max_operations: 10000 + max_bytes: 128MiB +producer: + max_buffered_bytes: 64MiB +execution: + max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. + max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. + mongodb: # Only valid with the mongodb driver; tune independently in each role. + max_concurrent_writes: 64 + max_concurrent_groups: 16 merge: - max_attempts: 3 # Revision-conflict attempts for Merge and folded conditional Put. + max_attempts: 3 # Revision-conflict attempts. lua: - timeout: 100ms # Maximum wall-clock time per Lua execution. - max_source_bytes: 64KiB # Maximum Lua program source bytes. - max_result_bytes: 16MiB # Current/input/result document bound, including expansion. - max_cached_programs: 256 # Compiled programs retained in this process's LRU cache. - max_instructions: 1000000 # Maximum VM instruction checkpoints per execution. + timeout: 100ms # Per-script execution protection, not an RPC deadline. + max_source_bytes: 64KiB + max_result_bytes: 16MiB # Per-document current/input/result bound. + max_cached_programs: 256 + max_instructions: 1000000 -shutdown_timeout: 15s # Grace period for RPC shutdown and database disconnect. +health: + address: ":8081" +prometheus: + enabled: false + address: ":9090" +shutdown_timeout: 15s -# Internal diagnostics; restart the process after changes. See ../docs/logging.md. logging: level: warn # Minimum severity for all components: debug, info, warn, error; default warn. console: # Standard error output; no log files. diff --git a/configs/gateway.yaml b/configs/gateway.yaml index c467393..37cc722 100644 --- a/configs/gateway.yaml +++ b/configs/gateway.yaml @@ -1,55 +1,37 @@ -# Gateway: route public RPCs to per-Store Engines; no database or Kafka configuration. -# Values are defaults unless marked as examples or derived defaults. -# Capacities are per process, not cluster-wide or hard RSS limits. +# Gateway runtime configuration. Unknown and wrong-role fields are rejected. # Byte sizes accept KiB/MiB/GiB or integer bytes; durations accept ms/s/m/h. -# Positive limits cannot be disabled with 0. Unknown fields are rejected. -# Validate offline: sink config check --config configs/gateway.yaml -# Restart Gateway after changing any settings, including Store routes. -mode: gateway # Required process role. +# Validate: sink config check --config configs/gateway.yaml +mode: gateway -memory: - # max_bytes: 256MiB # Omit to detect from runtime/container memory limits. - burst_percent: 10 # Reserved for completion; never used for new request admission. - wait_timeout: 2s # Capacity growth wait; a shorter caller deadline wins. - -# Public RPC and gRPC health listener. grpc: - address: ":8080" # TCP listen address. - max_receive_message_bytes: 64MiB # Maximum encoded inbound gRPC message. - max_send_message_bytes: 64MiB # Maximum encoded outbound gRPC message. + address: ":8080" + max_receive_message_bytes: 64MiB + max_send_message_bytes: 64MiB +memory: + # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. + burst_percent: 10 # Completion reserve, 1..99. + wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. +request: + max_operations: 1000 # Entire client request across all Stores. +forwarding: + routes: + - store: primary + target: dns:///engine-primary:8080 + tls: + insecure: true # Example trusted plaintext network. + server_name: "" + dns_refresh_interval: 30s + idle_timeout: 5m + max_connections: 256 + max_fanout: 8 -# Dedicated health listener; always available independently of Prometheus. health: - address: ":8081" # Listen address for /livez and /readyz; default :8081. - -# Separate optional Prometheus listener. + address: ":8081" prometheus: - enabled: false # Start /metrics on its own port; does not affect health; default false. - address: ":9090" # Metrics-only listen address, used only when enabled; default :9090. - -# Per-public-request limits, retained across all forwarded Store groups. -service: - # Per-original-request limits, including requests split/coalesced by Gateway/Engine. - request: - timeout: 30s # Overall unary/page deadline including waits; <=300s; caller may shorten. - max_read_bytes: 32MiB # Read/returned-write/native document budget; <=half gRPC send limit. - max_operations: 1000 # Maximum operations in one Read, Write, or Delete request. - -gateway: # Stateless routing, bounded forwarding, and lazy downstream connections. - routes: # Store routes are loaded only at startup; restart after changes. - - store: primary # Must match the Engine/Worker storage.name and public request Store. - target: dns:///engine-primary:8080 # Required gRPC target for this Store's Engines. - tls: # Downstream transport; identity checks do not replace authentication. - insecure: true # Example: trusted plaintext network. Default false requires a TLS endpoint/proxy. - server_name: "" # Optional TLS name override; leave empty when insecure is true. - dns_refresh_interval: 30s # Refresh DNS-backed Engine replica addresses. - idle_timeout: 5m # Evict unused downstream channels after this idle duration. - max_connections: 256 # Cached endpoint channels and discovery views; <=10000; active calls protected. - max_fanout: 8 # Concurrent Store groups per RPC; <=128; budget-sensitive groups serialize. - -shutdown_timeout: 15s # Grace period for draining public RPCs and closing connections. + enabled: false + address: ":9090" +shutdown_timeout: 15s -# Internal diagnostics; restart the process after changes. See ../docs/logging.md. logging: level: warn # Minimum severity for all components: debug, info, warn, error; default warn. console: # Standard error output; no log files. diff --git a/configs/stores/primary.yaml b/configs/stores/primary.yaml new file mode 100644 index 0000000..917e1e9 --- /dev/null +++ b/configs/stores/primary.yaml @@ -0,0 +1,28 @@ +# Shared Store identity and dependency policy. Pass with --store-config. +# This file is reused by Engine and Worker; restart both after policy changes. +name: primary +storage: + driver: mongodb # mongodb, elasticsearch, or opensearch. + mongodb: + uri: mongodb://mongodb:27017 # Example endpoint; required unless uri_file is set. + # uri_file: /etc/sink-secrets/mongodb-uri # Omit uri when using its file alternative. + metadata_field: __sink # Must match across roles and replicas. + # For elasticsearch/opensearch, replace mongodb with search: + # search: + # endpoints: [http://opensearch:9200] + # username_file: /etc/sink-secrets/search-username + # password_file: /etc/sink-secrets/search-password + # api_key_file: /etc/sink-secrets/search-api-key # Alternative to basic auth. +kafka: + enabled: true # Disable for synchronous-only Stores; Worker requires true. + brokers: [kafka:9092] + topic: + name: primary-mutations + partitions: 4 + replication_factor: 2 + min_insync_replicas: 2 + retention: 72h + max_record_bytes: 900KiB + dead_letter: + topic: primary-mutations.dlq # Defaults to .dlq. + retention: 720h diff --git a/configs/worker.yaml b/configs/worker.yaml index 0d323cf..daaaa48 100644 --- a/configs/worker.yaml +++ b/configs/worker.yaml @@ -1,106 +1,41 @@ -# Worker: consume and apply Kafka mutations for one Store; no business gRPC listener. -# Values are defaults unless marked as examples or derived defaults. -# Capacities are per process, not cluster-wide or hard RSS limits. +# Worker runtime configuration. Unknown and wrong-role fields are rejected. # Byte sizes accept KiB/MiB/GiB or integer bytes; durations accept ms/s/m/h. -# Positive limits cannot be disabled with 0. Unknown fields are rejected. -# Validate offline: sink config check --config configs/worker.yaml -# Engine must enable Kafka to publish mutations for this Worker. -mode: worker # Required process role. +# Validate: sink config check --config configs/worker.yaml --store-config configs/stores/primary.yaml +mode: worker -# One shared process pool; max_bytes is normally detected automatically. -memory: - # max_bytes: 256MiB # Optional explicit capacity, including completion reserve. - burst_percent: 10 # Completion reserve; 1..99; new requests use ordinary capacity. - wait_timeout: 2s # Response growth wait, bounded by the original request deadline. - -# Dedicated health listener; always available independently of Prometheus. -health: - address: ":8081" # Listen address for /livez and /readyz; default :8081. - -# Separate optional Prometheus listener. -prometheus: - enabled: false # Start /metrics on its own port; does not affect health; default false. - address: ":9090" # Metrics-only listen address, used only when enabled; default :9090. - -# Bind this process to one Store. Its replicas must use the same Store name. -storage: - name: primary # Required globally unique Store name; shared by all its replicas. - driver: mongodb # Required: mongodb, elasticsearch, or opensearch. - - # Credential-file alternatives below are optional; remove the matching inline key, - # even if empty, before enabling its _file variant. Paths must be absolute. - # Files contain 1 byte..1MiB of UTF-8 without NUL; spaces/newlines are used exactly. - # Files are read at startup and by config check; restart after credential rotation. - - # MongoDB driver settings. Replace with the search block for a search driver. - mongodb: - uri: mongodb://mongodb:27017 # Example; complete connection URI; required unless uri_file is set. - # uri_file: /etc/sink-secrets/mongodb-uri # Example mounted URI file; omit uri when enabled. - metadata_field: __sink # Internal revision field; hidden from clients; default __sink. - # metadata_field cannot be _id or contain a dot, dollar sign, or NUL. - max_concurrent_writes: 64 # Concurrent conditional writes per process; >0; default 64. - max_concurrent_groups: 16 # Concurrent collection groups shared by all calls; >0; default 16. - - # Search driver alternative: remove mongodb, set driver to elasticsearch or opensearch, - # and uncomment the search block. Leave nested file alternatives commented unless used. - # Basic auth requires both username and password; cannot be combined with an API key. - # search: # Backend endpoints and authentication for this Store only. - # endpoints: [http://opensearch:9200] # Example HTTP(S) URLs; at least one required; no default. - # username: "" # Optional basic-auth user; default empty; pair with password/password_file. - # # username_file: /etc/sink-secrets/search-username # Example user file; omit username. - # password: "" # Optional basic-auth password; default empty; pair with username/username_file. - # # password_file: /etc/sink-secrets/search-password # Example password file; omit password. - # api_key: "" # Optional API key; default empty; cannot be combined with basic auth. - # # api_key_file: /etc/sink-secrets/search-api-key # Example API-key file; omit api_key. - - # Kafka consumption is required; match the Engine Store, backend, and topic policy. - kafka: - enabled: true # Required in Worker mode; no default consumer without Kafka. - brokers: [kafka:9092] # Example bootstrap addresses; required when enabled. - # Source topic policy. Matching Engine/Worker replicas must use the same policy. - topic: - name: primary-mutations # Example; required when Kafka is enabled. - partitions: 4 # Exact source/DLQ partition count; changes need a drained migration. - replication_factor: 2 # Replicas per partition; requires enough Kafka brokers. - retention: 72h # Source topic retention; must be at least 1ms. - min_insync_replicas: 2 # Default min(2, replication_factor); cannot exceed it. - max_record_bytes: 900KiB # Encoded mutation plus key; <= producer buffer, <=64MiB. - # Consumption and retry settings for this Store. - consumer: - group_id: primary-workers # Example; required for Worker, default empty. - max_poll_records: 500 # Maximum records handled in one fetched batch. - processing_timeout: 20s # Backend time per batch; <=20s, plus bounded settlement. - # Temporary failures remain pending after a round; attempts do not discard them. - retry: - max_attempts: 10 # Attempts per processing round, not a message lifetime limit. - backoff: 100ms # Initial retry delay before jitter. - max_backoff: 10s # Delay cap before jitter; must be >= backoff. - # Permanent, malformed, and wrong-Store messages go to the dead-letter topic. - dead_letter: - topic: primary-mutations.dlq # Default .dlq; must differ from source. - retention: 720h # Independent DLQ retention (30 days); at least 1ms. - -# Limits for synchronous application of consumed mutations and Lua execution. -service: - # Limits per execution call made by the consumer. - request: - timeout: 30s # Execution deadline; consumer.processing_timeout may shorten it. - max_read_bytes: 32MiB # Merge snapshot/output budget per execution call; default 32MiB. - max_operations: 1000 # Caps consumer poll records and operations in each execution call. - - # Conditional mutation conflict handling and Lua limits. +# Only the managed capacity is configurable; reservation policy is internal. +# memory: +# max_bytes: 256MiB +consumer: + group_id: primary-workers + max_poll_records: 500 + processing_timeout: 20s # One processing round; unresolved records remain replayable. + retry: + max_attempts: 10 # Per round, not a message lifetime limit. + backoff: 100ms + max_backoff: 10s +execution: + max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. + max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. + mongodb: # Only valid with the mongodb driver; tune independently in each role. + max_concurrent_writes: 64 + max_concurrent_groups: 16 merge: - max_attempts: 3 # Revision-conflict attempts for Merge and folded conditional Put. + max_attempts: 3 # Revision-conflict attempts. lua: - timeout: 100ms # Maximum wall-clock time per Lua execution. - max_source_bytes: 64KiB # Maximum Lua program source bytes. - max_result_bytes: 16MiB # Current/input/result document bound, including expansion. - max_cached_programs: 256 # Compiled programs retained in this process's LRU cache. - max_instructions: 1000000 # Maximum VM instruction checkpoints per execution. + timeout: 100ms # Per-script execution protection, not an RPC deadline. + max_source_bytes: 64KiB + max_result_bytes: 16MiB # Per-document current/input/result bound. + max_cached_programs: 256 + max_instructions: 1000000 -shutdown_timeout: 15s # Grace period for consumer shutdown and database disconnect. +health: + address: ":8081" +prometheus: + enabled: false + address: ":9090" +shutdown_timeout: 15s -# Internal diagnostics; restart the process after changes. See ../docs/logging.md. logging: level: warn # Minimum severity for all components: debug, info, warn, error; default warn. console: # Standard error output; no log files. diff --git a/docs/batching.md b/docs/batching.md index b223915..d823927 100644 --- a/docs/batching.md +++ b/docs/batching.md @@ -8,11 +8,11 @@ is always active; every single-store `Read` uses this path. `Write` and mutations. Read, write, and delete have independent queues within each store. A slow batch therefore does not block another method or another store. -`service.batching` configures batch and queue limits; batching cannot be disabled. -Remove the former `service.batching.enabled` field from existing configurations. +`batching` configures batch and queue limits; batching cannot be disabled. +Remove the former `batching.enabled` field from existing configurations. The strict configuration parser rejects it as an unknown field for either value. -The first queued request starts `service.batching.max_wait`. +The first queued request starts `batching.max_wait`. Collection stops when that timer expires or adding another request would cross the operation or encoded-byte target. A single valid RPC larger than a batch target still runs alone. Automatic mutation batches combine only RPCs sharing diff --git a/docs/configuration-migration.md b/docs/configuration-migration.md index e0e8f24..5cd0d49 100644 --- a/docs/configuration-migration.md +++ b/docs/configuration-migration.md @@ -1,25 +1,34 @@ -# New-cluster deployment +# Configuration migration -This release uses a new URI-only record protocol and canonical URI Kafka keys. -Deploy matching Gateway, Engine, Worker and SDK builds in a separate cluster. -Use new Kafka topics and consumer groups; old mutation envelopes and scan -checkpoints are not accepted migration inputs. Perform the client cutover through -blue/green deployment after validating the new cluster. +This is a breaking configuration and private forwarding change. Deploy matching +Gateway and Engine versions. The public SDK protocol remains on Gateway; direct +public RPCs on Engine are no longer supported. Use a separate cluster for cutover +from an incompatible forwarding version; old and new Engines cannot share routes. -Use the current [configuration reference](configuration.md), -[component examples](../configs/README.md), and [runtime guide](store-isolation.md). -Only `gateway`, `engine` and `worker` modes are supported. Engine and Worker use -one `storage` object. Gateway uses inline `gateway.routes` and has no database configuration. -Store names use the lowercase syntax in [record addresses](record-addresses.md). -HTTP health endpoints always run at `health.address` (default `:8081`). -Prometheus `/metrics` uses its own `prometheus.address` (default `:9090`) and -requires `prometheus.enabled: true`. Move HTTP probes to the health port. +1. Extract `storage.name` to `name` in a Store file. Put database settings under + `storage` and shared Kafka settings under top-level `kafka` in that file. +2. Pass that same file to Engine and Worker with `--store-config`. Remove embedded + storage settings from their component files. Gateway takes only `--config`. +3. Move `gateway` to `forwarding`. Keep only `request.max_operations` on Gateway. +4. Move `service.batching` to Engine's `batching`; move `service.merge` to + `execution.merge`. Put MongoDB concurrency under `execution.mongodb`. +5. Move Kafka `producer` to Engine and `consumer` to Worker. Worker has no `grpc`, + `request`, `batching`, or `producer`, and `memory` accepts only `max_bytes`. +6. Remove `service.request.timeout` and response/read quotas. Caller contexts + control request lifetime; gRPC message limits bound transport responses. + Snapshot/output working sets use `execution.max_snapshot_bytes` and + `execution.max_output_bytes`. Remove old count-based execution/publish admission. -Move the old route file entries into `gateway.routes` and remove `routes_file`, -`reload_interval`, and each route's `state`. All listed routes are active. Restart -Gateway after changing configuration; there is no route hot reload. +```sh +sink config check --config configs/gateway.yaml +sink config check --config configs/engine.yaml --store-config configs/stores/primary.yaml +sink config check --config configs/worker.yaml --store-config configs/stores/primary.yaml +``` -Validate configurations offline with `sink config check --config FILE`, then -verify the seven public RPCs, key affinity and asynchronous processing against -isolated resources before switching clients. This repository change does not -perform deployment or database migration. +The [annotated examples](../configs/README.md) and [configuration reference](configuration.md) +cover the complete schema. The Chart mounts one shared Store ConfigMap into both +roles and passes the second argument. Chart 0.8 requires the matching Sink 0.19 +release; pre-release validation must override its image with this candidate build. +Existing Kafka data and Store identities need not change for this configuration +migration when the stored mutation protocol is already compatible. This change +performs no deployment or database migration. diff --git a/docs/configuration.md b/docs/configuration.md index 5ed28e7..a0caed5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,22 +1,23 @@ # Runtime configuration -Use the [Gateway / Engine / Worker guide](store-isolation.md) for the topology and routing configuration. Engine and Worker each require one `storage` object; Gateway accepts routes only. - -Sink reads all server runtime parameters from one YAML file. Pass its path when -starting the binary: +Use the [Gateway / Engine / Worker guide](store-isolation.md) for topology. +Each process reads a role configuration. Engine and Worker also read the same +Store configuration through an explicit command-line argument: ```shell -sink --config /etc/sink/config.yaml +sink --config configs/gateway.yaml +sink --config configs/engine.yaml --store-config configs/stores/primary.yaml +sink --config configs/worker.yaml --store-config configs/stores/primary.yaml ``` -The `--config` option is required for every Gateway, Engine, and Worker runtime. -`sink version` does not load a configuration file. `sink lua test` does not -require one, but accepts `--config` to apply the same `service.merge.lua` resource -limits without opening any configured backend. Environment variables such as -`SINK_MODE` and `SINK_MONGODB_URI` are not read by the server. - -Use `sink config check --config FILE` to validate the schema and limits without -connecting to dependencies or printing configured values. +Both files use standard YAML. There is no include syntax, merging, or field override +between them. Paths passed on the command line are relative to the working directory. +`--config` is required. `--store-config` is required for Engine/Worker and rejected +for Gateway. `sink config check` accepts the same arguments and validates both files +offline without opening dependencies or printing configured values. `sink version` +requires neither file. `sink lua test` can optionally load the pair to use +`execution.merge.lua` limits. Environment variables such as `SINK_MODE` and +`SINK_MONGODB_URI` are not read by the server. Configuration is loaded once during startup. Unknown fields, malformed YAML, multiple YAML documents, invalid Store names, invalid positive-integer @@ -24,29 +25,26 @@ values, and incompatible option combinations prevent the process from starting. Backend connections are lazy and dependency recovery is independent within each Engine or Worker. Kafka publication/consumption remains unavailable until topic policy has been reconciled and verified. Other Store processes are -independent. All configuration, including `gateway.routes`, is loaded only at +independent. All configuration, including `forwarding.routes`, is loaded only at startup. Restart after changes. Configuration files are limited to 4 MiB. ## Configuration layout -| Section | Responsibility | +| File / section | Responsibility | | --- | --- | -| `mode`, `shutdown_timeout` | Process role and shutdown | -| `grpc`, `health`, `prometheus` | Transport and observability listeners | -| `logging` | Default warn-level stderr logs and optional direct OTLP export; see [logging reference](logging.md) | -| `storage` | Engine/Worker Store identity, backend and Kafka path | -| `gateway` | Inline Store routes, forwarding capacity, connection cache and discovery | -| `service.request` | Request deadline, operation count, and returned document budget | -| `service.execution` | Storage execution capacity, with Scan sublimits | -| `service.publish` | Independent Kafka publication capacity | -| `service.batching` | Collection targets and waiting queue capacity | -| `service.merge` | Conflict attempts and Lua execution limits | - -Kafka settings are grouped into `topic`, `producer`, `consumer`, and -`dead_letter`. Time values use Go duration strings (`100ms`, `30s`, `72h`); -numeric time values and old field names are rejected. Defaults are resolved once -before the application opens resources. Omitted values use defaults; explicit -zero or negative limits fail validation. +| Role: `mode`, `shutdown_timeout` | Process role and graceful shutdown | +| Role: `grpc`, `health`, `prometheus`, `logging` | Listeners and observability; Worker has no `grpc` | +| Role: `memory` | Process capacity; Worker accepts only optional `max_bytes` | +| Gateway: `request.max_operations` | One public batch across all Stores | +| Gateway: `forwarding` | Store routes, discovery and fanout | +| Engine: `batching`, `producer` | Collection queues and Kafka publishing buffer | +| Worker: `consumer` | Group, polling, processing rounds and retries | +| Engine/Worker: `execution` | Snapshot/output working sets, MongoDB concurrency and Lua | +| Store: `name`, `storage`, `kafka` | Shared identity, database connection and Topic/DLQ policy | + +Request validation belongs to Gateway. Engine has no `request` section and accepts +only the private forwarding protocol and gRPC health. SDKs address Gateway. +Unknown fields and sections belonging to a different role are rejected. ### Human-readable values @@ -68,62 +66,37 @@ for the complete old-to-new mapping. The gRPC and SDK contracts are unchanged. ## One Store per Engine or Worker -Each Engine/Worker process binds to exactly one globally unique `storage.name`. -Choose MongoDB, Elasticsearch, or OpenSearch for that Store. Different Stores must -own different database targets; replicas of the same Store share its identity. -A request's Store must match the process binding before any side effect. +Each Engine/Worker process binds to the `name` in its Store file. Replicas of one +Store reuse the same file; independent Stores must own different database targets. +The Store file holds shared connection and delivery policy: ```yaml -mode: engine -grpc: - address: ":8080" -health: - address: ":8081" -prometheus: - enabled: false - address: ":9090" +name: catalog storage: - name: catalog driver: mongodb mongodb: uri: mongodb://mongodb:27017 metadata_field: __sink - kafka: - enabled: true - brokers: [kafka:9092] - topic: - name: catalog-mutations - partitions: 4 - replication_factor: 2 - retention: 72h - dead_letter: - topic: catalog-mutations.dlq -service: - request: - timeout: 30s - max_operations: 1000 -shutdown_timeout: 15s +kafka: + enabled: true + brokers: [kafka:9092] + topic: + name: catalog-mutations + partitions: 4 + replication_factor: 2 + dead_letter: + topic: catalog-mutations.dlq ``` Kafka is optional for Engine. Without it, synchronous calls remain available and -asynchronous mutations return per-operation `UNAVAILABLE`. Worker requires Kafka -and `storage.kafka.consumer.group_id`. Deploy Worker separately with the same -Store name, backend, topic and topic policy. - -Gateway handles cross-Store batches and preserves original result order, including -mixed asynchronous batches whose Stores have different publishing availability. -It does not load database or Kafka settings. - -Use the annotated example for the role you are configuring: - -- [Gateway](../configs/gateway.yaml): public request limits, forwarding, and connection discovery. -- [Engine](../configs/engine.yaml): one Store's backend, execution, batching, Lua, and optional Kafka publication. -- [Worker](../configs/worker.yaml): one Store's backend, consumption, execution, and Lua; no RPC listener or RPC batching. +asynchronous mutations return `UNAVAILABLE`. Worker requires Kafka plus a +`consumer.group_id` in its own role file. Group, retry and poll tuning never enter +the shared Store file. Engine's `producer` buffer tuning is also role-local. -Each component file loads directly without uncommenting another role's settings. -Engine and Worker include a small commented search-driver alternative to MongoDB. -Together the component examples cover all supported configuration fields. -Use the [quickstart](../examples/quickstart/README.md) to run all three components. +See the complete annotated [Gateway](../configs/gateway.yaml), +[Engine](../configs/engine.yaml), [Worker](../configs/worker.yaml) and +[Store](../configs/stores/primary.yaml) examples, or run the +[quickstart](../examples/quickstart/README.md). ## Address routing @@ -136,7 +109,7 @@ SDK examples, connection discovery, replica changes and deployment boundaries. ## Synchronous request batching -`service.batching` sets the collection targets and the per-method +`batching` sets the collection targets and the per-method `queue` limits. Batching is always active. See [batching behavior](batching.md) for queue admission, ordering, execution budgets, and completion boundaries. @@ -163,61 +136,78 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `health.address` | string | No | `:8081` | Any valid TCP listen address; empty uses the default | Always-on HTTP listener for `/livez` and `/readyz` in every role. | | `prometheus.enabled` | boolean | No | `false` | `true`, `false` | Start the separate metrics listener; health endpoints are unaffected. | | `prometheus.address` | string | No | `:9090` | Any valid TCP listen address; empty uses the default | Listen address for `/metrics`, used only when enabled. | -| `storage` | object | Engine/Worker | none | Exactly one storage object | The process-bound Store; forbidden in Gateway. | -| `storage.name` | string | Yes | none | Nonempty UTF-8 identity, at most 256 bytes | Globally unique Store name selected by `address.store`; all replicas of that Store use the same name. | +| `storage` | object | Store file | none | Exactly one storage object | Shared backend configuration. | +| `name` | string | Yes | none | Nonempty UTF-8 identity, at most 256 bytes | Globally unique Store name selected by `address.store`; all replicas of that Store use the same name. | | `storage.driver` | enum string | Yes | none | `mongodb`, `elasticsearch`, `opensearch` | Adapter used by this storage instance. See [Storage driver values](#storage-driver-values). | | `storage.mongodb.uri` | string | Conditionally | none | Valid MongoDB connection string | Required when the entry's driver is `mongodb`. | | `storage.mongodb.metadata_field` | string | No | `__sink` | Any valid MongoDB field except `_id`; cannot contain `.`, `$`, or a null byte | Reserved top-level field where Sink stores internal metadata such as the record revision; removed from documents returned to clients. | -| `storage.mongodb.max_concurrent_writes` | positive integer | No | `64` | Integer greater than `0` | Maximum concurrent MongoDB conditional writes. | -| `storage.mongodb.max_concurrent_groups` | positive integer | No | `16` | Integer greater than `0` | Maximum collection groups executed concurrently across all calls to one store. | +| `execution.mongodb.max_concurrent_writes` | positive integer | No | `64` | Integer greater than `0` | Maximum concurrent MongoDB conditional writes. | +| `execution.mongodb.max_concurrent_groups` | positive integer | No | `16` | Integer greater than `0` | Maximum collection groups executed concurrently across all calls to one store. | | `storage.search.endpoints` | list of strings | Conditionally | none | One or more HTTP(S) endpoints | Required for `elasticsearch` and `opensearch`. | | `storage.search.username` | string | Conditionally | empty | Any username accepted by the search service | Basic-auth username. Must be configured together with `password`. | | `storage.search.password` | string | Conditionally | empty | Any password accepted by the search service | Basic-auth password. Must be configured together with `username`. | | `storage.search.api_key` | string | No | empty | Any API key accepted by the search service | API key used instead of basic authentication. | -| `service.request.max_operations` | positive integer | No | `1000` | Integer greater than `0` | Maximum operation count accepted in one Read, Write, or Delete batch request. | -| `service.merge.max_attempts` | positive integer | No | `3` | Integer greater than `0` | Maximum revision-conflict attempts for Merge and folded conditional Put chains. | -| `service.batching.max_wait` | duration string | No | `2ms` | Positive Go duration within the bounds below | Maximum collection delay measured from the first request in a batch. | -| `service.batching.max_operations` | positive integer | No | `service.request.max_operations` | Integer from `1` through `service.request.max_operations` | Operation target for one automatically formed batch. | -| `service.batching.max_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Encoded-byte target for one automatically formed batch; one larger valid RPC still runs alone. | -| `service.batching.queue.max_operations` | positive integer | No | max(`10000`, `service.request.max_operations`) | Integer at least `service.request.max_operations` and `service.batching.max_operations` | Maximum operations waiting in each method queue. | -| `service.batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `service.batching.max_bytes` | Maximum encoded request bytes waiting in each method queue. | -| `service.merge.lua.timeout` | duration string | No | `100ms` | Positive Go duration within the bounds below | Maximum wall-clock duration of one Lua execution. | -| `service.merge.lua.max_source_bytes` | byte size | No | `64KiB` | Size greater than `0B` | Maximum Lua source size per merge operation. | -| `service.merge.lua.max_result_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Maximum input/current document and encoded result bytes; expanded output is also bounded before conversion. | -| `service.merge.lua.max_cached_programs` | positive integer | No | `256` | Integer greater than `0` | Maximum compiled Lua programs retained in the process-local LRU cache. | -| `service.merge.lua.max_instructions` | positive integer | No | `1000000` | Integer greater than `0` | Maximum VM instruction checkpoints per execution. | -| `storage.kafka` | object | No | absent | A store-specific Kafka configuration | Holds the asynchronous delivery and Topic-management policy. Kafka remains disabled unless `enabled` is `true`. | -| `storage.kafka.enabled` | boolean | No | `false` | `true`, `false` | Enables Kafka publication, consumption, and startup Topic reconciliation for this store. | -| `storage.kafka.brokers` | list of strings | Conditionally | none | One or more Kafka bootstrap addresses | Required when Kafka is enabled. Each store may use a different Kafka cluster. | -| `storage.kafka.topic.name` | string | Conditionally | none | Any valid Kafka topic name | Required when Kafka is enabled. The server publishes only mutations whose `address.store` selects this store. | -| `storage.kafka.consumer.group_id` | string | Conditionally | empty | Any valid Kafka consumer group ID | Required in `worker` mode; optional and unused in `engine` mode. | -| `storage.kafka.dead_letter.topic` | string | No | `.dlq` | Non-empty Kafka topic different from the source topic | Destination for malformed, cross-store, and permanent failures. Temporary failures remain in the source Topic. | -| `storage.kafka.topic.partitions` | positive integer | No | `4` | Integer from `1` through `2147483647` | Required partition count for both Topics. Any mismatch gates this store; changes require an explicit drained migration. | -| `storage.kafka.topic.replication_factor` | positive integer | No | `2` | Integer from `1` through `32767`, not exceeding available brokers | Required replica count for every partition of both Topics. Sink submits and waits for partition reassignment when it differs. | -| `storage.kafka.topic.retention` | duration string | No | `72h` (3 days) | Positive Go duration within the bounds below | Source Topic retention, at least `1ms`. DLQ retention is configured separately. | -| `storage.kafka.consumer.max_poll_records` | positive integer | No | `500` | Integer greater than `0` | Maximum number of this store's mutations handled in one consumer fetch batch. | -| `storage.kafka.consumer.retry.max_attempts` | positive integer | No | `10` | Integer greater than `0` | Maximum attempts per processing round. Temporary failures are retained and retried in later rounds. | -| `storage.kafka.consumer.retry.backoff` | duration string | No | `100ms` | Positive Go duration within the bounds below | Initial worker retry backoff before jitter for this store. | -| `storage.kafka.consumer.retry.max_backoff` | duration string | No | `10s` | Positive Go duration within the bounds below | Maximum worker retry backoff before jitter; at least `consumer.retry.backoff` and at most half the Go duration range for safe doubling. | +| `request.max_operations` | positive integer | No | `1000` | Integer greater than `0` | Maximum operation count accepted in one Read, Write, or Delete batch request. | +| `execution.merge.max_attempts` | positive integer | No | `3` | Integer greater than `0` | Maximum revision-conflict attempts for Merge and folded conditional Put chains. | +| `batching.max_wait` | duration string | No | `2ms` | Positive Go duration within the bounds below | Maximum collection delay measured from the first request in a batch. | +| `batching.max_operations` | positive integer | No | `1000` | Positive integer | Operation target for one automatically formed batch. | +| `batching.max_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Encoded-byte target for one automatically formed batch; one larger valid RPC still runs alone. | +| `batching.queue.max_operations` | positive integer | No | max(`10000`, `batching.max_operations`) | Integer at least `batching.max_operations` | Maximum operations waiting in each method queue. | +| `batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `batching.max_bytes` | Maximum encoded request bytes waiting in each method queue. | +| `execution.merge.lua.timeout` | duration string | No | `100ms` | Positive Go duration within the bounds below | Maximum wall-clock duration of one Lua execution. | +| `execution.merge.lua.max_source_bytes` | byte size | No | `64KiB` | Size greater than `0B` | Maximum Lua source size per merge operation. | +| `execution.merge.lua.max_result_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Maximum input/current document and encoded result bytes; expanded output is also bounded before conversion. | +| `execution.merge.lua.max_cached_programs` | positive integer | No | `256` | Integer greater than `0` | Maximum compiled Lua programs retained in the process-local LRU cache. | +| `execution.merge.lua.max_instructions` | positive integer | No | `1000000` | Integer greater than `0` | Maximum VM instruction checkpoints per execution. | +| `kafka` | object | No | absent | A store-specific Kafka configuration | Holds the asynchronous delivery and Topic-management policy. Kafka remains disabled unless `enabled` is `true`. | +| `kafka.enabled` | boolean | No | `false` | `true`, `false` | Enables Kafka publication, consumption, and startup Topic reconciliation for this store. | +| `kafka.brokers` | list of strings | Conditionally | none | One or more Kafka bootstrap addresses | Required when Kafka is enabled. Each store may use a different Kafka cluster. | +| `kafka.topic.name` | string | Conditionally | none | Any valid Kafka topic name | Required when Kafka is enabled. The server publishes only mutations whose `address.store` selects this store. | +| `consumer.group_id` | string | Conditionally | empty | Any valid Kafka consumer group ID | Required in `worker` mode; optional and unused in `engine` mode. | +| `kafka.dead_letter.topic` | string | No | `.dlq` | Non-empty Kafka topic different from the source topic | Destination for malformed, cross-store, and permanent failures. Temporary failures remain in the source Topic. | +| `kafka.topic.partitions` | positive integer | No | `4` | Integer from `1` through `2147483647` | Required partition count for both Topics. Any mismatch gates this store; changes require an explicit drained migration. | +| `kafka.topic.replication_factor` | positive integer | No | `2` | Integer from `1` through `32767`, not exceeding available brokers | Required replica count for every partition of both Topics. Sink submits and waits for partition reassignment when it differs. | +| `kafka.topic.retention` | duration string | No | `72h` (3 days) | Positive Go duration within the bounds below | Source Topic retention, at least `1ms`. DLQ retention is configured separately. | +| `consumer.max_poll_records` | positive integer | No | `500` | Integer greater than `0` | Maximum number of this store's mutations handled in one consumer fetch batch. | +| `consumer.retry.max_attempts` | positive integer | No | `10` | Integer greater than `0` | Maximum attempts per processing round. Temporary failures are retained and retried in later rounds. | +| `consumer.retry.backoff` | duration string | No | `100ms` | Positive Go duration within the bounds below | Initial worker retry backoff before jitter for this store. | +| `consumer.retry.max_backoff` | duration string | No | `10s` | Positive Go duration within the bounds below | Maximum worker retry backoff before jitter; at least `consumer.retry.backoff` and at most half the Go duration range for safe doubling. | | `shutdown_timeout` | duration string | No | `15s` | Positive Go duration within the bounds below | Maximum graceful-shutdown time for gRPC and MongoDB disconnect operations. | ### Reliability limits +There is no server-wide business request timeout and no default SDK Scan deadline. +Callers choose deadlines or wait until completion, cancellation or process shutdown. +`memory.wait_timeout` bounds capacity contention, `execution.merge.lua.timeout` +bounds a Lua invocation, and `consumer.processing_timeout` bounds a Worker processing +round; none of these is a whole-RPC deadline. + +There is no `request.max_response_bytes` or `request.max_read_bytes` knob. Gateway +uses `grpc.max_send_message_bytes` as the transport ceiling and reserves encoded +result overhead before forwarding. Returned documents share the remaining budget +across all Store groups of the original RPC. Oversized returned writes are rejected +before their own commit. Execution snapshots and candidate outputs have independent +Store-local working sets; they do not consume the public response allowance. + +| Execution setting | Default | Meaning | +| --- | --- | --- | +| `execution.max_snapshot_bytes` | `32MiB` | Conditional write snapshot working set | +| `execution.max_output_bytes` | `32MiB` | Candidate write output working set | + + All settings in this table are optional. Counts and bytes are positive integers; time values are positive duration strings such as `30s` or `2ms`. Limits are process-local; replica counts multiply capacity. Configure the same Kafka policy on servers and workers. | Setting | Default | Meaning | | --- | --- | --- | -| `service.request.timeout` | `30s` | Unary request timeout including batching queue wait and each Scan page; at most 300 seconds. A shorter caller deadline wins. | | `memory.max_bytes` | automatic | Shared managed capacity for this process; omit to use half the smallest detected Go/cgroup/host memory limit, or 256 MiB fallback. Explicit values must be at least 1 KiB. Not an RSS cap. | | `memory.burst_percent` | `10` | Completion reserve as a percentage of total capacity, from 1 to 99. New requests cannot enter through this reserve. | | `memory.wait_timeout` | `2s` | Maximum response/working-set growth wait. Caller deadlines take precedence. New requests fail immediately when ordinary capacity is unavailable. | -| `service.request.max_read_bytes` | min(`32MiB`, half gRPC send limit) | Per-original-RPC Read or returned-Write documents, conditional write snapshot/output per attempt, and native Execute response. Scan pages use the smaller of this limit and 4 MiB. Cannot exceed half the gRPC send limit. | -| `storage.kafka.dead_letter.retention` | `720h` | Independent DLQ retention, 30 days; at least `1ms` and bounded by Go duration range. | -| `storage.kafka.topic.min_insync_replicas` | min(`2`, replication factor) | Minimum ISR, at most replication factor. Publishers require all ISR acknowledgements. | -| `storage.kafka.topic.max_record_bytes` | `900KiB` | Encoded mutation envelope plus key, including expanded Lua source; at most 64 MiB and no larger than the producer buffer. Topic/producer batch limits include an extra 16 KiB for framing and DLQ headers. Broker/replica fetch limits must also support increases. | -| `storage.kafka.producer.max_buffered_bytes` | `64MiB` | Producer buffer capacity, at most 1 GiB. Full buffers return retryable resource exhaustion. | -| `storage.kafka.consumer.processing_timeout` | `20s` | Backend work per fetched batch, at most 20 seconds, followed by at most 5 seconds of offset/DLQ settlement. | +| `kafka.dead_letter.retention` | `720h` | Independent DLQ retention, 30 days; at least `1ms` and bounded by Go duration range. | +| `kafka.topic.min_insync_replicas` | min(`2`, replication factor) | Minimum ISR, at most replication factor. Publishers require all ISR acknowledgements. | +| `kafka.topic.max_record_bytes` | `900KiB` | Encoded mutation envelope plus key, including expanded Lua source; at most 64 MiB; Engine producer buffer must cover it. Topic/producer batch limits include an extra 16 KiB for framing and DLQ headers. Broker/replica fetch limits must also support increases. | +| `producer.max_buffered_bytes` | `64MiB` | Producer buffer capacity, at most 1 GiB. Full buffers return retryable resource exhaustion. | +| `consumer.processing_timeout` | `20s` | Backend work per fetched batch, at most 20 seconds, followed by at most 5 seconds of offset/DLQ settlement. | All request classes share the same process pool. Input, small completion envelopes and forwarding scratch space acquire ordinary capacity. Document @@ -241,17 +231,15 @@ ownership follows batching and gRPC buffers beyond handler return and cancellati See [memory admission](design/demand-based-admission.md) and [metrics/KEDA migration](observability.md#memory-capacity-and-keda). -Legacy `gateway.max_bytes`, `gateway.max_requests`, -`gateway.max_requests_per_store`, `service.execution` and `service.publish` -admission fields remain accepted and validated, but no longer control CLI -admission. Replace their sizing with `memory`. Batch queue bounds, logical RPC -limits, backend limits and deadlines remain active. Engine and Worker still -scale separately, and Gateway connection/fanout bounds still apply. +The former `service` sections and Gateway count/byte admission settings are +rejected. Use `memory` for process capacity and `batching.queue` for Engine queues. +Gateway/Engine expose `burst_percent` and `wait_timeout`; Worker uses internal +completion-reserve and wait defaults and exposes only optional `max_bytes`. MongoDB group concurrency is shared across concurrent calls. Sink sets `w=majority` and `journal=true` on its client, overriding weaker URI concerns; server selection is bounded to five seconds. Verify the deployment supports -these settings before upgrading. The service deadline bounds the whole request. +these settings before upgrading. The caller context controls the overall request lifetime. A response budget can yield partial results: an individual oversized document is a permanent resource failure; exhaustion caused by other records in the same original RPC @@ -263,7 +251,7 @@ mutation results must not be retried without business idempotence. | Value | Behavior | | --- | --- | | `gateway` | Routes public RPCs to per-Store Engines; see [isolated configuration](store-isolation.md). | -| `engine` | Opens the gRPC listener for exactly one `storage`, validates Store name, executes synchronously and publishes async mutations. | +| `engine` | Exposes private forwarding and health, validates the bound Store, executes and publishes async mutations. | | `worker` | Requires exactly one Kafka-enabled Store, consumes and applies its mutations locally without opening the gRPC listener. | ### Storage driver values diff --git a/docs/design/demand-based-admission.md b/docs/design/demand-based-admission.md index 5f41de8..7d77a1e 100644 --- a/docs/design/demand-based-admission.md +++ b/docs/design/demand-based-admission.md @@ -2,7 +2,7 @@ The application uses one process-local capacity pool for Gateway, Engine or Worker. Logical per-RPC document limits remain independent of allocation -ownership: increasing `service.request.max_read_bytes` no longer prepays that +ownership: increasing `grpc.max_send_message_bytes` no longer prepays that maximum for every ordinary request. ## Capacity and admission @@ -98,8 +98,7 @@ error, not zero load. Keep synchronous replica floors above zero and monitor process RSS separately. See [observability](../observability.md#memory-capacity-and-keda). The CLI always enables this policy. Older `gateway.max_bytes`, Gateway request -count fields, `service.execution` and `service.publish` admission fields are -accepted and validated for configuration compatibility but no longer gate -requests. Application assembly always supplies the shared pool. +count fields, `service.execution` and `service.publish` admission fields were previously +accepted for compatibility; the role-config schema now rejects them. Application assembly always supplies the shared pool. Batch queue limits, request/response size limits, deadlines and backend resource limits remain active. Migrate autoscaling away from the old reservation gauges. diff --git a/docs/design/store-isolated-architecture.md b/docs/design/store-isolated-architecture.md index c191a1f..9e035e6 100644 --- a/docs/design/store-isolated-architecture.md +++ b/docs/design/store-isolated-architecture.md @@ -81,7 +81,7 @@ DNS aliases, node addresses, or different credentials for the same database must A database target may contain multiple replica nodes. Existing namespace/dataset semantics remain unchanged and are not reinterpreted as Stores during this redesign. Use the globally unique Store name as the sole configured identity. Engine and Worker -bind it through `storage.name`; Gateway routes and public requests carry the same +bind it through Store file `name`; Gateway routes and public requests carry the same value as `store`. Replicas of one Store share that name, while different Stores must have different names. No additional database identifier is required. diff --git a/docs/design/store-isolated-validation.md b/docs/design/store-isolated-validation.md index 1568762..0341581 100644 --- a/docs/design/store-isolated-validation.md +++ b/docs/design/store-isolated-validation.md @@ -110,7 +110,7 @@ request mixes, and latency targets. ## Store name identity follow-up -The configured identity is now only `storage.name` (Gateway routes use `store`). +The configured identity is now only Store file `name` (Gateway routes use `store`). Local race tests cover duplicate route names, configuration changes taking effect only after restart, and rejection of old forwarding versions or mismatched envelope/operation Stores before writes. The public protobuf is diff --git a/docs/document-write-flow.md b/docs/document-write-flow.md index 6dbd053..deb54e4 100644 --- a/docs/document-write-flow.md +++ b/docs/document-write-flow.md @@ -105,7 +105,7 @@ The address maps as follows: | Request field | Meaning in this example | | --- | --- | -| URI Store `primary` | Select the Gateway route for `primary`; its Engine is bound to `storage.name = primary` | +| URI Store `primary` | Select the Gateway route for `primary`; its Engine is bound to Store file `name: primary` | | URI segment `catalog` | MongoDB database `catalog` | | URI segment `products` | MongoDB collection `products` | | `key = product-42` | MongoDB `_id`, using the string `product-42` | @@ -263,7 +263,7 @@ another writer has not changed the version since it was read. MongoDB uses Sink's internal revision field; search backends use `_seq_no` and `_primary_term`. On a definite revision conflict, Sink currently allows **3 attempts by default, including the initial attempt**, controlled by -`service.merge.max_attempts`. Exhaustion produces a retryable `CONFLICT`. +`execution.merge.max_attempts`. Exhaustion produces a retryable `CONFLICT`. A network timeout or lost acknowledgement is not a definite revision conflict and does not establish that the previous attempt had no effect. diff --git a/docs/lua-testing.md b/docs/lua-testing.md index 488f4ca..882e9a1 100644 --- a/docs/lua-testing.md +++ b/docs/lua-testing.md @@ -4,8 +4,8 @@ The `sink lua test` command runs merge programs locally with the same parser, compiler, restricted Lua environment, `sink.v1` functions, JSON/BSON bridge, fresh virtual machine, and default resource limits used by the Sink server. It does not start a server or connect to MongoDB, Elasticsearch, OpenSearch, or -Kafka, and it does not require a Sink configuration file. Pass `--config` when -production overrides `service.merge.lua` limits; the command validates the file and +Kafka, and it does not require a Sink configuration file. Pass `--config` and `--store-config` when +production overrides `execution.merge.lua` limits; the command validates the file and uses those limits without opening its configured backends. Use the same Sink binary version in development, CI, and production. This keeps @@ -57,7 +57,7 @@ output is canonical Extended JSON so BSON-specific types remain visible. | Flag | Required | Value and behavior | | --- | --- | --- | | `--script` | Yes | Lua source file. The chunk must return one merge function. | -| `--config` | No | Sink YAML configuration whose `service.merge.lua` limits are applied. No backend is opened. | +| `--config` | No | Role YAML configuration, paired with `--store-config`, whose `execution.merge.lua` limits are applied. No backend is opened. | | `--cases` | No | One `.yaml`/`.yml` case or a directory of cases. Cannot be combined with the document flags below. | | `--encoding` | Without `--cases` | `json` or `bson`. | | `--current` | No | Existing JSON or Extended JSON object. Omit to pass `nil` as `current`. | diff --git a/docs/merge-folding.md b/docs/merge-folding.md index a9343ab..2ba1456 100644 --- a/docs/merge-folding.md +++ b/docs/merge-folding.md @@ -93,7 +93,7 @@ explicit Reads. Releases containing this change must describe these semantics. A definite revision conflict restarts the whole snapshot-based chain, including previous conditional/Lua failures. Lua observation times remain fixed. The -existing service.merge.max_attempts limit also bounds folded conditional Put +existing execution.merge.max_attempts limit also bounds folded conditional Put chains. Exhaustion returns a retryable CONFLICT for every operation in that unresolved chain. An ambiguous transport failure, lost acknowledgement, or cancellation is not replayed internally. Existing business idempotence @@ -103,7 +103,7 @@ batch transactions. Reads share one backend observation per address but produce independent payload and revision copies. Every returned document, including repetitions, consumes the response budget in original input order. Deduplication therefore does not -allow repeated keys to bypass max_read_bytes. The backend's unique-document +allow repeated keys to bypass the public response allowance. The backend's unique-document read also remains bounded. Missing/error results are returned to all matching operations. diff --git a/docs/native-access.md b/docs/native-access.md index d3ba227..62a5fe1 100644 --- a/docs/native-access.md +++ b/docs/native-access.md @@ -397,7 +397,7 @@ Updated Go clients retry temporary Scan admission rejections carrying `reason="SCAN_ADMISSION_REJECTED"`. The same command and cursor are reused; successful pages are returned once. Default policy: three attempts with exponential backoff starting at 100 ms, capped at one second, and 20% jitter, -within a 30-second whole-page timeout. `ClientOptions.ScanRetry` and +with no default whole-page timeout. `ClientOptions.ScanRetry` and `ClientOptions.ScanTimeout` configure these limits; `MaxAttempts: 1` disables retries. Older servers without the detail keep single-attempt behavior. @@ -431,8 +431,8 @@ from changing the record. ## Limits, deadlines, and retries -Native RPCs use the shared process memory pool and `service.request.timeout`. -A shorter caller deadline wins. Between Scan calls there is no held request +Native RPCs use the shared process memory pool and honor the caller context. +`ClientOptions.ScanTimeout` is opt-in; zero adds no deadline. Between Scan calls there is no held request capacity or background cursor. New requests acquire ordinary memory capacity or fail immediately. Temporary @@ -444,15 +444,15 @@ growth may wait up to `memory.wait_timeout` (default 2s), bounded by the page's original deadline. Cancellation and deadline expiry retain `CANCELED` and `DEADLINE_EXCEEDED` respectively. See [memory admission](design/demand-based-admission.md). -Execute responses and returned Write documents share `service.request.max_read_bytes` +Execute responses and returned Write documents share the gRPC send ceiling semantics; returned-document budgets are per original RPC even after batching. -Count uses a separate backend response budget of min(`service.request.max_read_bytes`, +Count uses a separate backend response budget of min(the gRPC send ceiling, 256 KiB), enforced by the adapter as well as admission. MongoDB counts still reserve the driver's 48 MiB wire ceiling. This keeps small count responses from reserving a full document page. Output space is reserved before committing a returned write. A candidate that cannot fit fails before its own write; earlier operations may already be applied. -Scan pages use at most min(`service.request.max_read_bytes`, 4 MiB), with count and byte +Scan pages use at most min(the gRPC send ceiling, 4 MiB), with count and byte limits both enforced. A single oversized document fails with `RESOURCE_EXHAUSTED`. Search grows its backend response buffer against actual capacity, within the logical limit of two page budgets plus 64 KiB of metadata and the store response limit. It reduces the hit count diff --git a/docs/observability.md b/docs/observability.md index e1a0e34..06b8795 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -76,7 +76,7 @@ Sink metrics: | `sink_kafka_worker_delivery_seconds` | histogram | `store` | Oldest fetched-record age at source commit, including quarantined outcomes. | | `sink_kafka_worker_quarantined_total` | counter | `store` | Acknowledged DLQ publications, including replayed quarantine attempts. | -Engine and Worker Store labels use the single `storage.name` captured at startup. +Engine and Worker Store labels use the single Store file `name` captured at startup. Unknown or malformed request identities use bounded fallback labels instead of creating arbitrary time series. Gateway forwarding metrics use the configured route Store names; the Gateway splits cross-Store RPCs before Engine execution. diff --git a/docs/record-addresses.md b/docs/record-addresses.md index 92dfd72..183d678 100644 --- a/docs/record-addresses.md +++ b/docs/record-addresses.md @@ -79,7 +79,7 @@ original RPC's budgets; adding replicas does not multiply the return budget. Eligible independent groups run with bounded fanout. Native requests select an endpoint in round-robin order because they have no record key. -Connections are opened lazily per endpoint and reused. `gateway.max_connections` +Connections are opened lazily per endpoint and reused. `forwarding.max_connections` bounds active/idle endpoint channels and cached discovery entries; the DNS view for a Store cannot exceed that endpoint bound. Idle entries expire or are evicted under pressure. Active entries are not evicted. TLS verification uses the original diff --git a/docs/reliability.md b/docs/reliability.md index b7f44d5..87129a2 100644 --- a/docs/reliability.md +++ b/docs/reliability.md @@ -166,7 +166,7 @@ document payloads are not printed. Select a concrete range of 1–100 consecutiv records, at most 64 MiB. Missing/expired offsets fail the command. ```sh -sink dlq inspect --config /etc/sink/config.yaml --store primary \ +sink dlq inspect --config /etc/sink/worker.yaml --store-config /etc/sink/store.yaml --store primary \ --partition 0 --offset 12 --count 3 > dlq-inspection.jsonl ``` @@ -176,7 +176,7 @@ envelope/store before republishing, but publishing the selection is not atomic. It does not remove DLQ records or advance the worker's group offsets: ```sh -sink dlq replay --config /etc/sink/config.yaml --store primary \ +sink dlq replay --config /etc/sink/worker.yaml --store-config /etc/sink/store.yaml --store primary \ --partition 0 --offset 12 --count 3 > dlq-replay.jsonl ``` @@ -232,7 +232,7 @@ remain per original RPC. Final failed writes release their returned-document reservation; successful CAS retries retain only the final document's charge. Read/Write/Delete also reserve up to 1 KiB of failure text plus a result envelope per operation before execution or publishing. Failure messages are valid UTF-8, -limited to 1 KiB each, and further shortened according to `service.request.max_read_bytes` +limited to 1 KiB each, and further shortened according to the gRPC response ceiling per original RPC, retaining at least one byte to satisfy the client contract. Codes, retryability, and operation indexes remain intact. This error allowance is separate from the returned-document quota. diff --git a/docs/rolling-upgrades.md b/docs/rolling-upgrades.md index 7bdd6aa..4c7126c 100644 --- a/docs/rolling-upgrades.md +++ b/docs/rolling-upgrades.md @@ -1,5 +1,9 @@ # Rolling upgrades and replica changes +The role-config redesign changes the private forwarding version. Migrate from +0.18 and older through a separate matching Gateway/Engine cluster. The compatible +rolling procedure below does not apply across that boundary. + Treat discovery withdrawal and RPC draining as separate intervals. Sink keeps established channels when DNS refreshes, and Gateway retains one Engine address snapshot for each Store throughout an accepted public batch. Stopping an Engine @@ -37,7 +41,7 @@ The first inequality is an operational sizing model, not a hard bound during DNS outages. Include CoreDNS, NodeLocal DNSCache, forwarders and sidecars actually used on the path. A successful refresh may return an old cached answer; a failed lookup retains the last usable membership and follows resolver backoff. Lowering -`gateway.dns_refresh_interval` does not invalidate upstream caches or guarantee a +`forwarding.dns_refresh_interval` does not invalidate upstream caches or guarantee a maximum failed-lookup duration. For example, 5s publication + 30s cache + 30s refresh + 5s lookup + 30s request diff --git a/docs/store-isolation.md b/docs/store-isolation.md index 7cd0429..152e908 100644 --- a/docs/store-isolation.md +++ b/docs/store-isolation.md @@ -17,46 +17,40 @@ flowchart LR WorkerA --> DatabaseA ``` -Gateway exposes the existing seven public RPCs. Engine exposes the same RPCs for -its one Store, plus a private versioned forwarding RPC. Worker has no application +Gateway exposes the existing seven public RPCs. Engine exposes private versioned forwarding and health RPCs for its one Store. Worker has no application gRPC listener and calls the shared execution core directly. Asynchronous acceptance still means that Engine's Kafka producer received durable acknowledgement; Gateway never connects to Kafka or a database. ## Configuration -An Engine uses `mode: engine` and one `storage` mapping with `name`, `driver`, -and the existing driver/Kafka settings. Worker uses the same mapping with -`mode: worker`. The globally unique Store name is the only configured identity; -all Engine and Worker replicas of that Store use the same name. +Engine uses `mode: engine`; Worker uses `mode: worker`. Both receive the same +Store file through `--store-config`, holding `name`, `storage` and shared `kafka` +policy. Component tuning remains in their own `--config` files. A Store must have its own database target. Distinct URI aliases do not establish that two targets are different. Deployment inventory must enforce this globally. Engine checks the expected Store name on every forwarded call. It rejects an entire mismatched batch before storage or publishing. -Gateway uses `mode: gateway`, `grpc`, `health`, `prometheus`, `service.request`, and `gateway`: +Gateway uses `mode: gateway`, `grpc`, `health`, `prometheus`, `request`, and `forwarding`: | Setting | Default | Meaning | | --- | --- | --- | -| `gateway.routes` | required | Inline Store routes; every entry is active; restart after changes | -| `gateway.dns_refresh_interval` | `30s` | Refresh DNS even while existing connections are healthy | -| `gateway.idle_timeout` | `5m` | Close channels that have no active calls and remain idle | -| `gateway.max_connections` | `256` | Maximum cached gRPC channels, created lazily; a channel can have multiple backend connections | -| `gateway.max_requests` | `128` | Maximum admitted public requests | -| `gateway.max_requests_per_store` | `min(32, max_requests)` | Maximum active forwarded calls to one Store | -| `gateway.max_bytes` | `256MiB` | Logical input/envelope/response reservations for admitted requests | -| `gateway.max_fanout` | `8` | Maximum parallel Store calls per public batch | +| `forwarding.routes` | required | Inline Store routes; every entry is active; restart after changes | +| `forwarding.dns_refresh_interval` | `30s` | Refresh DNS even while existing connections are healthy | +| `forwarding.idle_timeout` | `5m` | Close channels that have no active calls and remain idle | +| `forwarding.max_connections` | `256` | Maximum cached gRPC channels, created lazily; a channel can have multiple backend connections | +| `forwarding.max_fanout` | `8` | Maximum parallel Store calls per public batch | These are process limits, not RSS guarantees or recommended resource requests. -Gateway reserves known input/result overhead; reads, returned documents and native -replies additionally reserve their configured maximum response allowance. Plain -puts, deletes and async acceptance do not reserve a full document response quota. -Size limits do not include all Go, gRPC, TLS or OS allocation overhead. +Gateway charges actual input/result allocations to `memory`; growing responses +share a bounded completion reserve. The gRPC send ceiling determines response +capacity. Clients control request deadlines; shutdown has its own bounded drain. Routes belong to the same Gateway configuration: ```yaml -gateway: +forwarding: routes: - store: primary target: dns:///primary-engine.example:443 @@ -72,7 +66,7 @@ inside the trusted service network. Store name checking does not provide client authentication or authorization. All configured routes are active. There is no route state flag or hot reload. -Add, remove, or update entries in `gateway.routes`, then restart Gateway. During +Add, remove, or update entries in `forwarding.routes`, then restart Gateway. During a rolling restart, each instance uses the configuration loaded at its own startup. Graceful shutdown drains accepted calls within the configured deadline; never replay a write automatically just because its connection closes. diff --git a/examples/kubernetes/engine-deployment.yaml b/examples/kubernetes/engine-deployment.yaml index d1eef85..3e86e78 100644 --- a/examples/kubernetes/engine-deployment.yaml +++ b/examples/kubernetes/engine-deployment.yaml @@ -42,7 +42,7 @@ spec: containers: - name: sink image: ghcr.io/batchstream/sink:REPLACE_WITH_TESTED_RELEASE - args: ["--config", "/etc/sink/sink.yaml"] + args: ["--config", "/etc/sink/sink.yaml", "--store-config", "/etc/sink/store.yaml"] resources: requests: cpu: "2" diff --git a/examples/kubernetes/sink.yaml b/examples/kubernetes/sink.yaml index 2f5b7d5..f7d0433 100644 --- a/examples/kubernetes/sink.yaml +++ b/examples/kubernetes/sink.yaml @@ -1,46 +1,16 @@ -# This is a template. Supply real backend configuration through your Secret -# management system. Sink reads this file directly; it does not expand ${ENV}. +# Engine component configuration. Mount store.yaml from your configuration Secret. mode: engine -grpc: - address: :8080 - max_receive_message_bytes: 64MiB - max_send_message_bytes: 64MiB -health: - address: ":8081" -prometheus: - enabled: true - address: :9090 -storage: - name: mongo - driver: mongodb - mongodb: - uri: mongodb://mongo-0.example.invalid:27017,mongo-1.example.invalid:27017,mongo-2.example.invalid:27017/?replicaSet=rs0 - max_concurrent_writes: 64 - max_concurrent_groups: 16 -service: - request: - timeout: 10s - # Small-document deployments can use 8388608 after validating aggregate - # per-RPC snapshot/result sizes; see docs/production-sizing.md. - max_read_bytes: 32MiB - max_operations: 1000 - execution: - max_requests: 128 - max_bytes: 512MiB - batching: - max_wait: 2ms - max_operations: 32 - max_bytes: 16MiB - queue: - max_operations: 10_000 - max_bytes: 64MiB - merge: - max_attempts: 3 - lua: - timeout: 100ms - max_source_bytes: 64KiB - max_result_bytes: 16MiB - max_cached_programs: 256 - max_instructions: 1_000_000 - +grpc: {address: ":8080", max_receive_message_bytes: 64MiB, max_send_message_bytes: 64MiB} +health: {address: ":8081"} +prometheus: {enabled: true, address: ":9090"} +execution: + max_snapshot_bytes: 32MiB + max_output_bytes: 32MiB + mongodb: {max_concurrent_writes: 64, max_concurrent_groups: 16} + merge: {max_attempts: 3} +batching: + max_wait: 2ms + max_operations: 32 + max_bytes: 16MiB + queue: {max_operations: 10000, max_bytes: 64MiB} shutdown_timeout: 30s diff --git a/examples/kubernetes/store.yaml b/examples/kubernetes/store.yaml new file mode 100644 index 0000000..7cbd313 --- /dev/null +++ b/examples/kubernetes/store.yaml @@ -0,0 +1,6 @@ +# Shared Engine/Worker Store configuration; example endpoint only. +name: mongo +storage: + driver: mongodb + mongodb: + uri: mongodb://mongo-0.example.invalid:27017,mongo-1.example.invalid:27017,mongo-2.example.invalid:27017/?replicaSet=rs0 diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md index 50ce65e..ffce808 100644 --- a/examples/quickstart/README.md +++ b/examples/quickstart/README.md @@ -28,10 +28,11 @@ Stop the example with: docker compose --env-file /dev/null -f examples/quickstart/compose.yaml down ``` -Store routes are configured in `gateway.yaml` under `gateway.routes`. All listed +Store routes are configured in `gateway.yaml` under `forwarding.routes`. Engine and Worker mount the same `store.yaml` and load it +with `--store-config`. All listed routes are active. Restart Gateway after changing its configuration. For another Store, add its own database target, Engine, Worker and route. Replicas -of one Store share that Store's `storage.name`; different Stores must not share a +of one Store share that Store file's `name`; different Stores must not share a database target. The example's plaintext route is for the local Compose network. See [configuration, migration and scaling](../../docs/store-isolation.md). diff --git a/examples/quickstart/compose.yaml b/examples/quickstart/compose.yaml index efeb4e6..dc9943b 100644 --- a/examples/quickstart/compose.yaml +++ b/examples/quickstart/compose.yaml @@ -59,9 +59,10 @@ services: context: ../.. dockerfile: Dockerfile init: true - command: [--config, /etc/sink/engine.yaml] + command: [--config, /etc/sink/engine.yaml, --store-config, /etc/sink/store.yaml] volumes: - ./engine.yaml:/etc/sink/engine.yaml:ro + - ./store.yaml:/etc/sink/store.yaml:ro ports: - 127.0.0.1:18081:8081 - 127.0.0.1:19091:9090 @@ -73,9 +74,10 @@ services: pull_policy: never image: sink-quickstart:local init: true - command: [--config, /etc/sink/worker.yaml] + command: [--config, /etc/sink/worker.yaml, --store-config, /etc/sink/store.yaml] volumes: - ./worker.yaml:/etc/sink/worker.yaml:ro + - ./store.yaml:/etc/sink/store.yaml:ro ports: - 127.0.0.1:18082:8081 - 127.0.0.1:19092:9090 diff --git a/examples/quickstart/engine.yaml b/examples/quickstart/engine.yaml index 9128491..093a032 100644 --- a/examples/quickstart/engine.yaml +++ b/examples/quickstart/engine.yaml @@ -1,59 +1,40 @@ mode: engine - grpc: address: ":8080" max_receive_message_bytes: 64MiB max_send_message_bytes: 64MiB - -health: - address: ":8081" -prometheus: - enabled: true - address: ":9090" - -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongo:27017/?replicaSet=rs0 +memory: + # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. + burst_percent: 10 # Completion reserve, 1..99. + wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. +batching: + max_wait: 2ms # Collection delay, not an execution timeout. + max_operations: 1000 + max_bytes: 16MiB + queue: + max_operations: 10000 + max_bytes: 128MiB +producer: + max_buffered_bytes: 64MiB +execution: + max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. + max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. + mongodb: # Only valid with the mongodb driver; tune independently in each role. max_concurrent_writes: 64 max_concurrent_groups: 16 - kafka: - enabled: true - brokers: - - kafka:19092 - topic: - name: sink-mutations - partitions: 4 - replication_factor: 1 - retention: 72h - consumer: - group_id: sink-quickstart - max_poll_records: 500 - retry: - max_attempts: 10 - backoff: 100ms - max_backoff: 10s - - dead_letter: - topic: sink-mutations.dlq -service: - request: - max_operations: 1000 - batching: - max_wait: 2ms - max_operations: 1000 - max_bytes: 16MiB - queue: - max_operations: 10_000 - max_bytes: 128MiB merge: - max_attempts: 3 + max_attempts: 3 # Revision-conflict attempts. lua: - timeout: 100ms + timeout: 100ms # Per-script execution protection, not an RPC deadline. max_source_bytes: 64KiB - max_result_bytes: 16MiB + max_result_bytes: 16MiB # Per-document current/input/result bound. max_cached_programs: 256 - max_instructions: 1_000_000 + max_instructions: 1000000 +health: + address: ":8081" +prometheus: + enabled: true + address: ":9090" shutdown_timeout: 15s +logging: {level: warn} diff --git a/examples/quickstart/gateway.yaml b/examples/quickstart/gateway.yaml index 457e71c..2cae75c 100644 --- a/examples/quickstart/gateway.yaml +++ b/examples/quickstart/gateway.yaml @@ -1,25 +1,30 @@ mode: gateway grpc: address: ":8080" -health: - address: ":8081" -prometheus: - enabled: true - address: ":9090" -gateway: - routes: # Store routes are loaded only at startup; restart after changes. + max_receive_message_bytes: 64MiB + max_send_message_bytes: 64MiB +memory: + # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. + burst_percent: 10 # Completion reserve, 1..99. + wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. +request: + max_operations: 1000 # Entire client request across all Stores. +forwarding: + routes: - store: primary target: dns:///engine:8080 tls: - insecure: true # Local Compose network; use TLS termination in production. + insecure: true # Example trusted plaintext network. + server_name: "" dns_refresh_interval: 30s idle_timeout: 5m max_connections: 256 - max_requests: 128 - max_bytes: 256MiB max_fanout: 8 -service: - request: - timeout: 30s - max_operations: 1000 - max_read_bytes: 32MiB + +health: + address: ":8081" +prometheus: + enabled: true + address: ":9090" +shutdown_timeout: 15s +logging: {level: warn} diff --git a/examples/quickstart/store.yaml b/examples/quickstart/store.yaml new file mode 100644 index 0000000..9a33fb8 --- /dev/null +++ b/examples/quickstart/store.yaml @@ -0,0 +1,28 @@ +# Shared Store identity and dependency policy. Pass with --store-config. +# This file is reused by Engine and Worker; restart both after policy changes. +name: primary +storage: + driver: mongodb # mongodb, elasticsearch, or opensearch. + mongodb: + uri: mongodb://mongo:27017/?replicaSet=rs0 # Example endpoint; required unless uri_file is set. + # uri_file: /etc/sink-secrets/mongodb-uri # Omit uri when using its file alternative. + metadata_field: __sink # Must match across roles and replicas. + # For elasticsearch/opensearch, replace mongodb with search: + # search: + # endpoints: [http://opensearch:9200] + # username_file: /etc/sink-secrets/search-username + # password_file: /etc/sink-secrets/search-password + # api_key_file: /etc/sink-secrets/search-api-key # Alternative to basic auth. +kafka: + enabled: true # Disable for synchronous-only Stores; Worker requires true. + brokers: [kafka:19092] + topic: + name: sink-mutations + partitions: 4 + replication_factor: 1 + min_insync_replicas: 1 + retention: 72h + max_record_bytes: 900KiB + dead_letter: + topic: sink-mutations.dlq # Defaults to .dlq. + retention: 720h diff --git a/examples/quickstart/worker.yaml b/examples/quickstart/worker.yaml index f1cdf75..cbadcf9 100644 --- a/examples/quickstart/worker.yaml +++ b/examples/quickstart/worker.yaml @@ -1,59 +1,34 @@ mode: worker - -grpc: - address: ":8080" - max_receive_message_bytes: 64MiB - max_send_message_bytes: 64MiB - -health: - address: ":8081" -prometheus: - enabled: true - address: ":9090" - -storage: - name: primary - driver: mongodb - mongodb: - uri: mongodb://mongo:27017/?replicaSet=rs0 +# Only the managed capacity is configurable; reservation policy is internal. +# memory: +# max_bytes: 256MiB +consumer: + group_id: primary-workers + max_poll_records: 500 + processing_timeout: 20s # One processing round; unresolved records remain replayable. + retry: + max_attempts: 10 # Per round, not a message lifetime limit. + backoff: 100ms + max_backoff: 10s +execution: + max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. + max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. + mongodb: # Only valid with the mongodb driver; tune independently in each role. max_concurrent_writes: 64 max_concurrent_groups: 16 - kafka: - enabled: true - brokers: - - kafka:19092 - topic: - name: sink-mutations - partitions: 4 - replication_factor: 1 - retention: 72h - consumer: - group_id: sink-quickstart - max_poll_records: 500 - retry: - max_attempts: 10 - backoff: 100ms - max_backoff: 10s - - dead_letter: - topic: sink-mutations.dlq -service: - request: - max_operations: 1000 - batching: - max_wait: 2ms - max_operations: 1000 - max_bytes: 16MiB - queue: - max_operations: 10_000 - max_bytes: 128MiB merge: - max_attempts: 3 + max_attempts: 3 # Revision-conflict attempts. lua: - timeout: 100ms + timeout: 100ms # Per-script execution protection, not an RPC deadline. max_source_bytes: 64KiB - max_result_bytes: 16MiB + max_result_bytes: 16MiB # Per-document current/input/result bound. max_cached_programs: 256 - max_instructions: 1_000_000 + max_instructions: 1000000 +health: + address: ":8081" +prometheus: + enabled: true + address: ":9090" shutdown_timeout: 15s +logging: {level: warn} From d7055820d6c80ad00ccd5f68507996dc31737976 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 04:40:24 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=93=8C=20ci:=20qualify=20the=20redesi?= =?UTF-8?q?gned=20configuration=20with=20matching=20fixtures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94c9915..bc861e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@e084c611c2ce6efb2e2175cbcfc931dd497c9811 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@5dfdb00edbf1de4758bdab8f4e136248ff1b6d92 with: - suite_ref: e084c611c2ce6efb2e2175cbcfc931dd497c9811 + suite_ref: 5dfdb00edbf1de4758bdab8f4e136248ff1b6d92 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} From cdce37c690d56b5741d9479088393676fa678f02 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 05:31:50 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(config):=20sh?= =?UTF-8?q?are=20Kafka=20policy=20and=20default=20minimum=20ISR=20to=20one?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- cmd/sink/dead_letter_command.go | 4 ++-- cmd/sink/dead_letter_command_test.go | 6 ++--- configs/stores/primary.yaml | 11 +++++----- docs/configuration-migration.md | 5 +++++ docs/configuration.md | 15 ++++++++----- docs/store-isolation.md | 2 +- examples/quickstart/store.yaml | 11 +++++----- internal/app/kafka.go | 18 +++++++-------- internal/config/config.go | 27 ++++++++++------------- internal/config/config_test.go | 5 ++++- internal/config/file.go | 26 +++++++++------------- internal/config/kafka.go | 19 ++++++++-------- internal/config/limits_test.go | 33 ++++++++++++++++++++++++++-- internal/config/load.go | 4 ++-- internal/queue/kafka/topics.go | 2 +- internal/queue/kafka/topics_test.go | 2 +- 17 files changed, 113 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc861e2..b491ddf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@5dfdb00edbf1de4758bdab8f4e136248ff1b6d92 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@0e7f42584ea67371e812774faf37840b5385e960 with: - suite_ref: 5dfdb00edbf1de4758bdab8f4e136248ff1b6d92 + suite_ref: 0e7f42584ea67371e812774faf37840b5385e960 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/cmd/sink/dead_letter_command.go b/cmd/sink/dead_letter_command.go index 57b20ad..d66fc5d 100644 --- a/cmd/sink/dead_letter_command.go +++ b/cmd/sink/dead_letter_command.go @@ -54,7 +54,7 @@ func runDeadLetterCommand(args []string, stdout io.Writer, stderr io.Writer) err } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - selection := queuekafka.DeadLetterRange{Brokers: selected.Kafka.Brokers, Topic: selected.Kafka.DeadLetter.Topic, + selection := queuekafka.DeadLetterRange{Brokers: selected.Kafka.Brokers, Topic: selected.Kafka.DeadLetter.Name, Partition: int32(*partition), Offset: *offset, Count: *count} records, err := queuekafka.ReadDeadLetters(ctx, selection) if err != nil { @@ -67,7 +67,7 @@ func runDeadLetterCommand(args []string, stdout io.Writer, stderr io.Writer) err return err } opts := queuekafka.PublisherOptions{Brokers: selected.Kafka.Brokers, Topic: selected.Kafka.Topic.Name, - MaxRecordBytes: selected.Kafka.Topic.MaxRecordBytes, MaxBufferedBytes: selected.Kafka.Producer.MaxBufferedBytes} + MaxRecordBytes: selected.Kafka.MaxRecordBytes, MaxBufferedBytes: selected.Kafka.Producer.MaxBufferedBytes} publisher, err := queuekafka.NewPublisher(opts) if err != nil { diff --git a/cmd/sink/dead_letter_command_test.go b/cmd/sink/dead_letter_command_test.go index 1cf91be..9716608 100644 --- a/cmd/sink/dead_letter_command_test.go +++ b/cmd/sink/dead_letter_command_test.go @@ -76,12 +76,12 @@ storage: %skafka: enabled: true brokers: [%q] + replication_factor: 1 + min_insync_replicas: 1 topic: name: source - replication_factor: 1 - min_insync_replicas: 1 dead_letter: - topic: source.dlq + name: source.dlq `, driver, backend, cluster.ListenAddrs()[0]) sharedPath := writeConfig(t, contents) configPath := writeConfig(t, "mode: engine\n") diff --git a/configs/stores/primary.yaml b/configs/stores/primary.yaml index 917e1e9..4ba1630 100644 --- a/configs/stores/primary.yaml +++ b/configs/stores/primary.yaml @@ -16,13 +16,14 @@ storage: kafka: enabled: true # Disable for synchronous-only Stores; Worker requires true. brokers: [kafka:9092] + # Shared by the source and dead-letter Topics. + partitions: 4 + replication_factor: 2 + min_insync_replicas: 1 + max_record_bytes: 900KiB topic: name: primary-mutations - partitions: 4 - replication_factor: 2 - min_insync_replicas: 2 retention: 72h - max_record_bytes: 900KiB dead_letter: - topic: primary-mutations.dlq # Defaults to .dlq. + name: primary-mutations.dlq # Defaults to .dlq. retention: 720h diff --git a/docs/configuration-migration.md b/docs/configuration-migration.md index 5cd0d49..14b3302 100644 --- a/docs/configuration-migration.md +++ b/docs/configuration-migration.md @@ -18,6 +18,11 @@ from an incompatible forwarding version; old and new Engines cannot share routes control request lifetime; gRPC message limits bound transport responses. Snapshot/output working sets use `execution.max_snapshot_bytes` and `execution.max_output_bytes`. Remove old count-based execution/publish admission. +7. Move Kafka `topic.partitions`, `topic.replication_factor`, + `topic.min_insync_replicas`, and `topic.max_record_bytes` directly under `kafka`; + they apply to both Topics. Rename `dead_letter.topic` to `dead_letter.name`. + Each Topic retains its own name and retention. `min_insync_replicas` now defaults + to `1`; set it explicitly if a higher minimum ISR is required. ```sh sink config check --config configs/gateway.yaml diff --git a/docs/configuration.md b/docs/configuration.md index a0caed5..f25309c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -164,9 +164,9 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `kafka.brokers` | list of strings | Conditionally | none | One or more Kafka bootstrap addresses | Required when Kafka is enabled. Each store may use a different Kafka cluster. | | `kafka.topic.name` | string | Conditionally | none | Any valid Kafka topic name | Required when Kafka is enabled. The server publishes only mutations whose `address.store` selects this store. | | `consumer.group_id` | string | Conditionally | empty | Any valid Kafka consumer group ID | Required in `worker` mode; optional and unused in `engine` mode. | -| `kafka.dead_letter.topic` | string | No | `.dlq` | Non-empty Kafka topic different from the source topic | Destination for malformed, cross-store, and permanent failures. Temporary failures remain in the source Topic. | -| `kafka.topic.partitions` | positive integer | No | `4` | Integer from `1` through `2147483647` | Required partition count for both Topics. Any mismatch gates this store; changes require an explicit drained migration. | -| `kafka.topic.replication_factor` | positive integer | No | `2` | Integer from `1` through `32767`, not exceeding available brokers | Required replica count for every partition of both Topics. Sink submits and waits for partition reassignment when it differs. | +| `kafka.dead_letter.name` | string | No | `.dlq` | Non-empty Kafka topic different from the source topic | Destination for malformed, cross-store, and permanent failures. Temporary failures remain in the source Topic. | +| `kafka.partitions` | positive integer | No | `4` | Integer from `1` through `2147483647` | Required partition count for both Topics. Any mismatch gates this store; changes require an explicit drained migration. | +| `kafka.replication_factor` | positive integer | No | `2` | Integer from `1` through `32767`, not exceeding available brokers | Required replica count for every partition of both Topics. Sink submits and waits for partition reassignment when it differs. | | `kafka.topic.retention` | duration string | No | `72h` (3 days) | Positive Go duration within the bounds below | Source Topic retention, at least `1ms`. DLQ retention is configured separately. | | `consumer.max_poll_records` | positive integer | No | `500` | Integer greater than `0` | Maximum number of this store's mutations handled in one consumer fetch batch. | | `consumer.retry.max_attempts` | positive integer | No | `10` | Integer greater than `0` | Maximum attempts per processing round. Temporary failures are retained and retried in later rounds. | @@ -204,11 +204,16 @@ counts multiply capacity. Configure the same Kafka policy on servers and workers | `memory.burst_percent` | `10` | Completion reserve as a percentage of total capacity, from 1 to 99. New requests cannot enter through this reserve. | | `memory.wait_timeout` | `2s` | Maximum response/working-set growth wait. Caller deadlines take precedence. New requests fail immediately when ordinary capacity is unavailable. | | `kafka.dead_letter.retention` | `720h` | Independent DLQ retention, 30 days; at least `1ms` and bounded by Go duration range. | -| `kafka.topic.min_insync_replicas` | min(`2`, replication factor) | Minimum ISR, at most replication factor. Publishers require all ISR acknowledgements. | -| `kafka.topic.max_record_bytes` | `900KiB` | Encoded mutation envelope plus key, including expanded Lua source; at most 64 MiB; Engine producer buffer must cover it. Topic/producer batch limits include an extra 16 KiB for framing and DLQ headers. Broker/replica fetch limits must also support increases. | +| `kafka.min_insync_replicas` | `1` | Minimum ISR for both Topics, at most replication factor. Publishers require all current ISR acknowledgements; the default permits writes with one in-sync replica during broker maintenance. | +| `kafka.max_record_bytes` | `900KiB` | Encoded mutation envelope plus key, including expanded Lua source; at most 64 MiB; Engine producer buffer must cover it. Topic/producer batch limits include an extra 16 KiB for framing and DLQ headers. Broker/replica fetch limits must also support increases. | | `producer.max_buffered_bytes` | `64MiB` | Producer buffer capacity, at most 1 GiB. Full buffers return retryable resource exhaustion. | | `consumer.processing_timeout` | `20s` | Backend work per fetched batch, at most 20 seconds, followed by at most 5 seconds of offset/DLQ settlement. | +`kafka.partitions`, `replication_factor`, `min_insync_replicas`, and +`max_record_bytes` apply to both the source and dead-letter Topics. Sink explicitly +reconciles this shared policy rather than relying on broker defaults. `kafka.topic` +and `kafka.dead_letter` each contain only `name` and `retention`, which are independent. + All request classes share the same process pool. Input, small completion envelopes and forwarding scratch space acquire ordinary capacity. Document copies and response encoding acquire capacity using known sizes; increasing the diff --git a/docs/store-isolation.md b/docs/store-isolation.md index 152e908..c408df0 100644 --- a/docs/store-isolation.md +++ b/docs/store-isolation.md @@ -107,7 +107,7 @@ forward Store groups concurrently within `max_fanout`. The extra network hop and serial budget-sensitive groups have a latency cost; benchmark your workload. Gateway never automatically replays Write, Delete or Execute. Complete responses -from healthy Stores are retained when another Store fails. A lost mutation reply +from healthy Stores are retained when another Store fails. A lost Engine mutation reply is represented as a failed operation with `retryable=false`, since the effect may already exist. A missing budget settlement consumes the entire grant; it cannot be reused by another Store. A proven local rejection before dispatch consumes diff --git a/examples/quickstart/store.yaml b/examples/quickstart/store.yaml index 9a33fb8..814579c 100644 --- a/examples/quickstart/store.yaml +++ b/examples/quickstart/store.yaml @@ -16,13 +16,14 @@ storage: kafka: enabled: true # Disable for synchronous-only Stores; Worker requires true. brokers: [kafka:19092] + # Shared by the source and dead-letter Topics. + partitions: 4 + replication_factor: 1 + min_insync_replicas: 1 + max_record_bytes: 900KiB topic: name: sink-mutations - partitions: 4 - replication_factor: 1 - min_insync_replicas: 1 retention: 72h - max_record_bytes: 900KiB dead_letter: - topic: sink-mutations.dlq # Defaults to .dlq. + name: sink-mutations.dlq # Defaults to .dlq. retention: 720h diff --git a/internal/app/kafka.go b/internal/app/kafka.go index 40412f0..610cc0c 100644 --- a/internal/app/kafka.go +++ b/internal/app/kafka.go @@ -17,14 +17,14 @@ func (app *Application) configureKafka(observed *sinkmetrics.Metrics) error { } topicOptions := queuekafka.TopicOptions{ Brokers: configured.Kafka.Brokers, - Topics: []string{configured.Kafka.Topic.Name, configured.Kafka.DeadLetter.Topic}, - Partitions: configured.Kafka.Topic.Partitions, - ReplicationFactor: configured.Kafka.Topic.ReplicationFactor, + Topics: []string{configured.Kafka.Topic.Name, configured.Kafka.DeadLetter.Name}, + Partitions: configured.Kafka.Partitions, + ReplicationFactor: configured.Kafka.ReplicationFactor, Retention: configured.Kafka.Topic.Retention, - DeadLetterTopic: configured.Kafka.DeadLetter.Topic, + DeadLetterTopic: configured.Kafka.DeadLetter.Name, DeadLetterRetention: configured.Kafka.DeadLetter.Retention, - MinInSyncReplicas: configured.Kafka.Topic.MinInSyncReplicas, - MaxRecordBytes: configured.Kafka.Topic.MaxRecordBytes, + MinInSyncReplicas: configured.Kafka.MinInSyncReplicas, + MaxRecordBytes: configured.Kafka.MaxRecordBytes, } app.topics = queuekafka.NewTopicManager(topicOptions) if app.config.Mode != config.ModeEngine { @@ -32,7 +32,7 @@ func (app *Application) configureKafka(observed *sinkmetrics.Metrics) error { } publisherOptions := queuekafka.PublisherOptions{ Store: configured.Name, Brokers: configured.Kafka.Brokers, Topics: app.topics, - MaxRecordBytes: configured.Kafka.Topic.MaxRecordBytes, + MaxRecordBytes: configured.Kafka.MaxRecordBytes, MaxBufferedBytes: configured.Kafka.Producer.MaxBufferedBytes, Topic: configured.Kafka.Topic.Name, Metrics: observed, } @@ -58,10 +58,10 @@ func (app *Application) configureWorker(server *service.Server, observed *sinkme workerOptions := queuekafka.WorkerOptions{ ShutdownTimeout: loaded.ShutdownTimeout, Topics: app.topics, ProcessingTimeout: configured.Kafka.Consumer.ProcessingTimeout, - MaxRecordBytes: configured.Kafka.Topic.MaxRecordBytes, + MaxRecordBytes: configured.Kafka.MaxRecordBytes, Brokers: configured.Kafka.Brokers, Store: configured.Name, Topic: configured.Kafka.Topic.Name, GroupID: configured.Kafka.Consumer.GroupID, - DeadLetterTopic: configured.Kafka.DeadLetter.Topic, Handler: processor, + DeadLetterTopic: configured.Kafka.DeadLetter.Name, Handler: processor, MaxPollRecords: configured.Kafka.Consumer.MaxPollRecords, MaxRetryAttempts: configured.Kafka.Consumer.Retry.MaxAttempts, RetryBackoff: configured.Kafka.Consumer.Retry.Backoff, diff --git a/internal/config/config.go b/internal/config/config.go index ee0d17a..9253d1a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -145,21 +145,21 @@ type Search struct { } type Kafka struct { - Enabled bool - Brokers []string - Topic Topic - Producer Producer - Consumer Consumer - DeadLetter DeadLetter -} - -type Topic struct { - Name string + Enabled bool + Brokers []string Partitions int ReplicationFactor int - Retention time.Duration MinInSyncReplicas int MaxRecordBytes int + Topic Topic + DeadLetter Topic + Producer Producer + Consumer Consumer +} + +type Topic struct { + Name string + Retention time.Duration } type Producer struct { @@ -179,11 +179,6 @@ type Retry struct { MaxBackoff time.Duration } -type DeadLetter struct { - Topic string - Retention time.Duration -} - // Gateway owns only routing and bounded forwarding resources. type Gateway struct { MaxRequestsPerStore int diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 64aae06..037ce19 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -72,7 +72,10 @@ func TestComponentDefaults(t *testing.T) { t.Fatal(err) } kafka := worker.Storage.Kafka - if kafka.DeadLetter.Topic != "mutations.dlq" || kafka.Consumer.MaxPollRecords != 500 || kafka.Consumer.ProcessingTimeout != 20*time.Second || kafka.Consumer.Retry.MaxAttempts != 10 { + if kafka.Partitions != 4 || kafka.ReplicationFactor != 2 || kafka.MinInSyncReplicas != 1 || kafka.MaxRecordBytes != 900<<10 || kafka.Topic.Retention != 72*time.Hour || kafka.DeadLetter.Retention != 720*time.Hour { + t.Fatal("shared Kafka defaults changed") + } + if kafka.DeadLetter.Name != "mutations.dlq" || kafka.Consumer.MaxPollRecords != 500 || kafka.Consumer.ProcessingTimeout != 20*time.Second || kafka.Consumer.Retry.MaxAttempts != 10 { t.Fatal("consumer defaults changed") } } diff --git a/internal/config/file.go b/internal/config/file.go index a7f7705..6a8dbbc 100644 --- a/internal/config/file.go +++ b/internal/config/file.go @@ -107,20 +107,19 @@ type searchFile struct { } type kafkaFile struct { - Enabled bool `yaml:"enabled"` - Brokers []string `yaml:"brokers"` - Topic topicFile `yaml:"topic"` - - DeadLetter deadLetterFile `yaml:"dead_letter"` + Enabled bool `yaml:"enabled"` + Brokers []string `yaml:"brokers"` + Partitions *int `yaml:"partitions"` + ReplicationFactor *int `yaml:"replication_factor"` + MinInSyncReplicas *int `yaml:"min_insync_replicas"` + MaxRecordBytes *byteSize `yaml:"max_record_bytes"` + Topic topicFile `yaml:"topic"` + DeadLetter topicFile `yaml:"dead_letter"` } type topicFile struct { - Name string `yaml:"name"` - Partitions *int `yaml:"partitions"` - ReplicationFactor *int `yaml:"replication_factor"` - Retention *time.Duration `yaml:"retention"` - MinInSyncReplicas *int `yaml:"min_insync_replicas"` - MaxRecordBytes *byteSize `yaml:"max_record_bytes"` + Name string `yaml:"name"` + Retention *time.Duration `yaml:"retention"` } type producerFile struct { @@ -140,11 +139,6 @@ type retryFile struct { MaxBackoff *time.Duration `yaml:"max_backoff"` } -type deadLetterFile struct { - Topic string `yaml:"topic"` - Retention *time.Duration `yaml:"retention"` -} - type gatewayFile struct { DNSRefreshInterval *time.Duration `yaml:"dns_refresh_interval"` Routes []Route `yaml:"routes"` diff --git a/internal/config/kafka.go b/internal/config/kafka.go index d814ec5..c6c0113 100644 --- a/internal/config/kafka.go +++ b/internal/config/kafka.go @@ -11,17 +11,16 @@ func resolveKafka(prefix string, file kafkaFile, v *validator) Kafka { var loaded Kafka loaded.Enabled = file.Enabled loaded.Brokers = nonEmptyValues(file.Brokers) + loaded.Partitions = v.bounded(prefix+".partitions", file.Partitions, 4, 1<<31-1) + loaded.ReplicationFactor = v.bounded(prefix+".replication_factor", file.ReplicationFactor, 2, 1<<15-1) + loaded.MinInSyncReplicas = v.bounded(prefix+".min_insync_replicas", file.MinInSyncReplicas, 1, loaded.ReplicationFactor) + loaded.MaxRecordBytes = v.bytes(prefix+".max_record_bytes", file.MaxRecordBytes, 900<<10, 64<<20) topic := &loaded.Topic topic.Name = strings.TrimSpace(file.Topic.Name) - topic.Partitions = v.bounded(prefix+".topic.partitions", file.Topic.Partitions, 4, 1<<31-1) - topic.ReplicationFactor = v.bounded(prefix+".topic.replication_factor", file.Topic.ReplicationFactor, 2, 1<<15-1) topic.Retention = v.duration(prefix+".topic.retention", file.Topic.Retention, 72*time.Hour) - topic.MinInSyncReplicas = v.bounded(prefix+".topic.min_insync_replicas", file.Topic.MinInSyncReplicas, min(2, topic.ReplicationFactor), topic.ReplicationFactor) - - topic.MaxRecordBytes = v.bytes(prefix+".topic.max_record_bytes", file.Topic.MaxRecordBytes, 900<<10, 64<<20) - loaded.DeadLetter.Topic = strings.TrimSpace(file.DeadLetter.Topic) - if topic.Name != "" && loaded.DeadLetter.Topic == "" { - loaded.DeadLetter.Topic = topic.Name + ".dlq" + loaded.DeadLetter.Name = strings.TrimSpace(file.DeadLetter.Name) + if topic.Name != "" && loaded.DeadLetter.Name == "" { + loaded.DeadLetter.Name = topic.Name + ".dlq" } loaded.DeadLetter.Retention = v.duration(prefix+".dead_letter.retention", file.DeadLetter.Retention, 720*time.Hour) if topic.Retention < time.Millisecond { @@ -81,8 +80,8 @@ func validateKafkaResources(loaded Config) error { if loaded.Mode == ModeWorker && kafka.Consumer.GroupID == "" { return errors.New("consumer.group_id is required in worker mode") } - if kafka.DeadLetter.Topic == kafka.Topic.Name { - return errors.New("kafka.dead_letter.topic must differ from topic.name") + if kafka.DeadLetter.Name == kafka.Topic.Name { + return errors.New("kafka.dead_letter.name must differ from kafka.topic.name") } return nil } diff --git a/internal/config/limits_test.go b/internal/config/limits_test.go index f9a443d..d096f30 100644 --- a/internal/config/limits_test.go +++ b/internal/config/limits_test.go @@ -3,6 +3,7 @@ package config import ( "strings" "testing" + "time" ) func TestExecutionBudgetsAreIndependentOfTransport(t *testing.T) { @@ -42,8 +43,10 @@ func TestRejectInvalidRoleLimits(t *testing.T) { func TestKafkaSharedPolicyValidation(t *testing.T) { for _, kafka := range []string{ "enabled: true", "enabled: true\n brokers: [localhost:1]", "topic: {retention: 1ns}", "dead_letter: {retention: 1ns}", - "topic: {partitions: 0}", "topic: {replication_factor: 1, min_insync_replicas: 2}", "topic: {max_record_bytes: 65MiB}", - "enabled: true\n brokers: [localhost:1]\n topic: {name: a}\n dead_letter: {topic: a}", + "partitions: 0", "replication_factor: 1\n min_insync_replicas: 2", "max_record_bytes: 65MiB", "min_insync_replicas: 0", + "enabled: true\n brokers: [localhost:1]\n topic: {name: a}\n dead_letter: {name: a}", + "topic: {partitions: 4}", "topic: {replication_factor: 2}", "topic: {min_insync_replicas: 1}", "topic: {max_record_bytes: 900KiB}", + "dead_letter: {topic: a.dlq}", "dead_letter: {partitions: 4}", "consumer: {}", "producer: {}", } { if _, err := Decode(strings.NewReader("mode: engine"), strings.NewReader(minimalStorage+"kafka:\n "+kafka)); err == nil { @@ -59,3 +62,29 @@ func TestKafkaSharedPolicyValidation(t *testing.T) { } } } + +func TestKafkaSharedPolicyLoading(t *testing.T) { + shared := minimalStorage + `kafka: + enabled: true + brokers: [localhost:9092] + partitions: 8 + replication_factor: 3 + min_insync_replicas: 2 + max_record_bytes: 2MiB + topic: {name: mutations, retention: 48h} + dead_letter: {name: rejected, retention: 240h} +` + for _, component := range []string{"mode: engine", "mode: worker\nconsumer: {group_id: workers}"} { + loaded, err := Decode(strings.NewReader(component), strings.NewReader(shared)) + if err != nil { + t.Fatal(err) + } + kafka := loaded.Storage.Kafka + if kafka.Partitions != 8 || kafka.ReplicationFactor != 3 || kafka.MinInSyncReplicas != 2 || kafka.MaxRecordBytes != 2<<20 { + t.Fatalf("shared Kafka policy not loaded for %s: %+v", component, kafka) + } + if kafka.Topic.Name != "mutations" || kafka.Topic.Retention != 48*time.Hour || kafka.DeadLetter.Name != "rejected" || kafka.DeadLetter.Retention != 240*time.Hour { + t.Fatalf("independent Topic settings not loaded: %+v", kafka) + } + } +} diff --git a/internal/config/load.go b/internal/config/load.go index a541ad9..b7ed240 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -167,8 +167,8 @@ func resolve(file configFile, shared storeFile) (Config, error) { if file.Producer != nil && !configured.Kafka.Enabled { v.reject(errors.New("producer requires Kafka enabled in Store config")) } - if loaded.Mode == ModeEngine && configured.Kafka.Topic.MaxRecordBytes > configured.Kafka.Producer.MaxBufferedBytes { - v.reject(errors.New("producer.max_buffered_bytes must cover kafka.topic.max_record_bytes")) + if loaded.Mode == ModeEngine && configured.Kafka.MaxRecordBytes > configured.Kafka.Producer.MaxBufferedBytes { + v.reject(errors.New("producer.max_buffered_bytes must cover kafka.max_record_bytes")) } if file.Execution != nil && file.Execution.MongoDB != (mongoExecutionFile{}) && configured.Driver != DriverMongoDB { v.reject(errors.New("execution.mongodb requires the mongodb driver")) diff --git a/internal/queue/kafka/topics.go b/internal/queue/kafka/topics.go index 542a6e8..0b2d1db 100644 --- a/internal/queue/kafka/topics.go +++ b/internal/queue/kafka/topics.go @@ -38,7 +38,7 @@ func EnsureTopics(ctx context.Context, opts TopicOptions) error { opts.DeadLetterRetention = 30 * 24 * time.Hour } if opts.MinInSyncReplicas == 0 { - opts.MinInSyncReplicas = min(2, opts.ReplicationFactor) + opts.MinInSyncReplicas = 1 } if opts.MaxRecordBytes == 0 { opts.MaxRecordBytes = defaultMaxRecordBytes diff --git a/internal/queue/kafka/topics_test.go b/internal/queue/kafka/topics_test.go index f1deb4b..5724975 100644 --- a/internal/queue/kafka/topics_test.go +++ b/internal/queue/kafka/topics_test.go @@ -83,7 +83,7 @@ func TestEnsureTopicsCreatesAndReconcilesTopicSettings(t *testing.T) { if resourceErr != nil { t.Fatalf("DescribeTopicConfigs(%q) error = %v", topic, resourceErr) } - for key, expected := range map[string]string{"min.insync.replicas": "2", "unclean.leader.election.enable": "false", "cleanup.policy": "delete"} { + for key, expected := range map[string]string{"min.insync.replicas": "1", "unclean.leader.election.enable": "false", "cleanup.policy": "delete"} { if actual := topicConfig(resource, key); actual != expected { t.Fatalf("%s %s=%s, want %s", topic, key, actual, expected) } From c1c326e766a851c5e37fd7f79016f5a20c14c132 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 05:53:48 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20perf:=20default=20auto?= =?UTF-8?q?matic=20batches=20to=2032=20operations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- configs/engine.yaml | 2 +- docs/batching.md | 28 +++++++++++++++++++++++- docs/configuration.md | 2 +- examples/quickstart/engine.yaml | 2 +- internal/config/config_test.go | 2 +- internal/config/service.go | 2 +- internal/service/batching_server.go | 5 +++-- internal/service/batching_server_test.go | 14 ++++++++++-- internal/service/options.go | 5 +---- 10 files changed, 50 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b491ddf..fe12769 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@0e7f42584ea67371e812774faf37840b5385e960 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@4c99c2bfd436db60a2845ed0731b8bf8c5b4557c with: - suite_ref: 0e7f42584ea67371e812774faf37840b5385e960 + suite_ref: 4c99c2bfd436db60a2845ed0731b8bf8c5b4557c sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/configs/engine.yaml b/configs/engine.yaml index b7e715c..6ec47c9 100644 --- a/configs/engine.yaml +++ b/configs/engine.yaml @@ -13,7 +13,7 @@ memory: wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. batching: max_wait: 2ms # Collection delay, not an execution timeout. - max_operations: 1000 + max_operations: 32 # Measured small-document starting point; not the public RPC limit. max_bytes: 16MiB queue: max_operations: 10000 diff --git a/docs/batching.md b/docs/batching.md index d823927..f573263 100644 --- a/docs/batching.md +++ b/docs/batching.md @@ -60,7 +60,8 @@ new single-store request that would cross its queue's limit fails with gRPC `RESOURCE_EXHAUSTED` and is not applied. Requests canceled before dispatch are omitted. Once a batch is dispatched, other live callers in that batch continue even if one caller cancels. Once all callers cancel, execution is cancelled too. -Execution is capped by the server request timeout even without caller deadlines. +Execution follows the callers' deadlines and cancellation; there is no default +whole-request timeout. Dispatched micro-batches acquire known working allocations from the shared `memory` pool. Response growth has priority over new arrivals and may borrow its completion reserve. Request entry fails if ordinary capacity is unavailable; @@ -82,3 +83,28 @@ scheduling, and serialization overhead and are therefore more efficient when the caller already has several records available. See the [configuration reference](configuration.md) for all settings. + +## Default selection + +`batching.max_operations` defaults to **32**, with a **2 ms** collection wait. +These are a starting point for small documents and modest concurrency, based on +the [historical Kubernetes measurements](https://github.com/batchstream/sink/blob/ec7264f25690088aac767a027d61b17753d08d99/docs/production-sizing.md#observed-small-document-ceilings) +and their [raw results](https://github.com/batchstream/sink/blob/ec7264f25690088aac767a027d61b17753d08d99/benchmarks/kubernetes/results/eks-arm64.csv). +At 128 clients, the 32-operation target improved both throughput and P99 over +128 for MongoDB and OpenSearch. At 512 clients, 128 performed better. These +measurements used the earlier shared-server architecture and fixed CPU/memory +budgets; they do not establish a universal optimum for the current Engine. + +The batch target is independent of Gateway's **1,000-operation** public RPC +limit and each method's **10,000-operation** waiting queue. A valid explicit +RPC with more than 32 operations executes alone; it is not rejected or split +into different public requests by this default change. + +`memory.burst_percent` remains **10**, the smallest tested completion reserve +that finished every admitted request in the +[allocator saturation experiment](https://github.com/batchstream/sink-production-suite/blob/main/benchmarks/memory-admission/README.md). +The remaining queue, byte, backend-concurrency and Kafka-consumer defaults are +capacity or protection settings without a comparable general tuning result. +Historical workload-specific response caps and GC settings are not carried over +as universal defaults. Tune these separately against actual document sizes, +resource limits and latency targets. diff --git a/docs/configuration.md b/docs/configuration.md index f25309c..542a2bd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -150,7 +150,7 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `request.max_operations` | positive integer | No | `1000` | Integer greater than `0` | Maximum operation count accepted in one Read, Write, or Delete batch request. | | `execution.merge.max_attempts` | positive integer | No | `3` | Integer greater than `0` | Maximum revision-conflict attempts for Merge and folded conditional Put chains. | | `batching.max_wait` | duration string | No | `2ms` | Positive Go duration within the bounds below | Maximum collection delay measured from the first request in a batch. | -| `batching.max_operations` | positive integer | No | `1000` | Positive integer | Operation target for one automatically formed batch. | +| `batching.max_operations` | positive integer | No | `32` | Positive integer | Operation target for one automatically formed batch; a larger valid RPC still executes alone. See [default selection](batching.md#default-selection). | | `batching.max_bytes` | byte size | No | `16MiB` | Size greater than `0B` | Encoded-byte target for one automatically formed batch; one larger valid RPC still runs alone. | | `batching.queue.max_operations` | positive integer | No | max(`10000`, `batching.max_operations`) | Integer at least `batching.max_operations` | Maximum operations waiting in each method queue. | | `batching.queue.max_bytes` | byte size | No | max(`128MiB`, `grpc.max_receive_message_bytes`) | Size at least `grpc.max_receive_message_bytes` and `batching.max_bytes` | Maximum encoded request bytes waiting in each method queue. | diff --git a/examples/quickstart/engine.yaml b/examples/quickstart/engine.yaml index 093a032..dc87b9f 100644 --- a/examples/quickstart/engine.yaml +++ b/examples/quickstart/engine.yaml @@ -9,7 +9,7 @@ memory: wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. batching: max_wait: 2ms # Collection delay, not an execution timeout. - max_operations: 1000 + max_operations: 32 # Measured small-document starting point; not the public RPC limit. max_bytes: 16MiB queue: max_operations: 10000 diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 037ce19..4b6cf01 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -54,7 +54,7 @@ func TestComponentDefaults(t *testing.T) { if engine.Service.Request.Timeout != 0 || engine.Service.Request.MaxOperations != 0 || engine.Service.Request.MaxReadBytes != 0 { t.Fatal("Engine acquired public request limits") } - if engine.Service.Batching.MaxOperations != 1000 || engine.Service.Batching.MaxWait != 2*time.Millisecond || engine.Service.Batching.Queue.MaxBytes != 128<<20 { + if engine.Service.Batching.MaxOperations != 32 || engine.Service.Batching.MaxWait != 2*time.Millisecond || engine.Service.Batching.Queue.MaxBytes != 128<<20 { t.Fatal("batch defaults changed") } if engine.Service.Execution.MaxSnapshotBytes != 32<<20 || engine.Service.Execution.MaxOutputBytes != 32<<20 || engine.Service.Merge.Lua.MaxResultBytes != 16<<20 { diff --git a/internal/config/service.go b/internal/config/service.go index ac2469f..b048c1e 100644 --- a/internal/config/service.go +++ b/internal/config/service.go @@ -42,7 +42,7 @@ func resolveService(file configFile, grpc GRPC, v *validator) Service { func resolveBatching(file batchingFile, grpc GRPC, v *validator) Batching { var loaded Batching loaded.MaxWait = v.duration("batching.max_wait", file.MaxWait, 2*time.Millisecond) - loaded.MaxOperations = v.integer("batching.max_operations", file.MaxOperations, 1000) + loaded.MaxOperations = v.integer("batching.max_operations", file.MaxOperations, 32) loaded.MaxBytes = v.bytes("batching.max_bytes", file.MaxBytes, 16<<20, math.MaxInt) loaded.Queue.MaxOperations = v.integer("batching.queue.max_operations", file.Queue.MaxOperations, max(10_000, loaded.MaxOperations)) loaded.Queue.MaxBytes = v.bytes("batching.queue.max_bytes", file.Queue.MaxBytes, max(128<<20, grpc.MaxReceiveMessageBytes), math.MaxInt) diff --git a/internal/service/batching_server.go b/internal/service/batching_server.go index 589048b..3ed60ca 100644 --- a/internal/service/batching_server.go +++ b/internal/service/batching_server.go @@ -19,6 +19,7 @@ import ( const ( defaultBatchMaxWait = 2 * time.Millisecond + defaultBatchMaxOperations = 32 defaultBatchMaxBytes = 16 << 20 defaultBatchMaxQueuedOperations = 10_000 defaultBatchMaxQueuedBytes = 128 << 20 @@ -124,7 +125,7 @@ func normalizeBatchingOptions(server *Server, opts BatchingOptions) (BatchingOpt opts.MaxWait = defaultBatchMaxWait } if opts.MaxOperations == 0 { - opts.MaxOperations = defaultMaxOperations + opts.MaxOperations = defaultBatchMaxOperations } if opts.MaxBytes == 0 { opts.MaxBytes = defaultBatchMaxBytes @@ -141,7 +142,7 @@ func normalizeBatchingOptions(server *Server, opts BatchingOptions) (BatchingOpt } if opts.MaxQueuedOperations < opts.MaxOperations { var empty BatchingOptions - return empty, errors.New("create synchronous batching server: queued operation limit must cover one server request and one batch") + return empty, errors.New("create synchronous batching server: queued operation limit must cover one batch") } if opts.MaxQueuedBytes < opts.MaxBytes { var empty BatchingOptions diff --git a/internal/service/batching_server_test.go b/internal/service/batching_server_test.go index 4017e03..b641b36 100644 --- a/internal/service/batching_server_test.go +++ b/internal/service/batching_server_test.go @@ -128,7 +128,17 @@ func TestNewBatchingServerValidatesLimits(t *testing.T) { if err != nil { t.Fatalf("NewBatchingServer() error = %v", err) } + // The smaller automatic batch target must not narrow the public RPC limit. + request := &sink.ReadRequest{Operations: make([]*sink.ReadOperation, 1000)} + for index := range request.Operations { + operation := &sink.ReadOperation{Address: protoAddress(fmt.Sprintf("record-%d", index))} + request.Operations[index] = operation + } + response, readErr := valid.Read(t.Context(), request) valid.Close() + if readErr != nil || len(response.GetResults()) != 1000 { + t.Fatalf("default batching rejected a valid explicit RPC: %v", readErr) + } tests := []struct { name string @@ -147,10 +157,10 @@ func TestNewBatchingServerValidatesLimits(t *testing.T) { }, }, { - name: "queue cannot hold one service request", + name: "queue cannot hold one automatic batch", server: core, options: service.BatchingOptions{ - MaxQueuedOperations: 999, + MaxQueuedOperations: 31, }, }, { diff --git a/internal/service/options.go b/internal/service/options.go index b128ad9..2e65f5c 100644 --- a/internal/service/options.go +++ b/internal/service/options.go @@ -12,10 +12,7 @@ import ( "github.com/liran/sink/internal/storage" ) -const ( - defaultMaxOperations = 1000 - defaultMaxMergeAttempts = 3 -) +const defaultMaxMergeAttempts = 3 type Options struct { Memory *capacity.Pool From cfe48c26c316d02b4cfc74dc3188567c4b08c7b6 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 06:57:38 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20replace=20?= =?UTF-8?q?allocation=20accounting=20with=20process=20memory=20watermarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 +- configs/engine.yaml | 11 +- configs/gateway.yaml | 6 +- configs/stores/primary.yaml | 2 + configs/worker.yaml | 12 +- deploy/reliability-alerts.yaml | 8 +- docs/batching.md | 38 +- docs/configuration-migration.md | 11 +- docs/configuration.md | 76 ++- docs/design/demand-based-admission.md | 190 ++++--- docs/native-access.md | 14 +- docs/observability.md | 92 ++-- docs/production-sizing.md | 18 +- docs/rolling-upgrades.md | 16 +- docs/store-isolation.md | 24 +- examples/autoscaling/memory-keda.yaml | 13 +- examples/kubernetes/sink.yaml | 2 - examples/quickstart/engine.yaml | 11 +- examples/quickstart/gateway.yaml | 6 +- examples/quickstart/store.yaml | 2 + examples/quickstart/worker.yaml | 12 +- gen/forward/forward.pb.go | 36 +- gen/forward/forward_vtproto.pb.go | 81 --- internal/app/app.go | 11 +- internal/app/kafka.go | 2 +- internal/app/logging.go | 6 - internal/app/logging_test.go | 10 +- internal/app/memory.go | 83 +++ internal/app/memory_test.go | 67 +++ internal/app/service.go | 7 +- internal/app/transport.go | 7 - internal/capacity/detect.go | 9 +- internal/capacity/detect_test.go | 32 ++ internal/capacity/guard.go | 143 ++++++ internal/capacity/guard_test.go | 142 ++++++ internal/capacity/metrics.go | 111 ++-- internal/capacity/pool.go | 297 ----------- internal/capacity/pool_test.go | 296 ----------- internal/capacity/reader.go | 44 -- internal/capacity/scope.go | 178 ------- internal/config/bytes_test.go | 2 - internal/config/config.go | 55 +- internal/config/config_test.go | 16 +- internal/config/file.go | 24 +- internal/config/gateway.go | 3 - internal/config/limits_test.go | 33 +- internal/config/load.go | 15 +- internal/config/memory_test.go | 9 +- internal/config/service.go | 5 - internal/config/storage.go | 5 + internal/engine/memory_test.go | 50 ++ internal/engine/server.go | 12 +- internal/engine/stream.go | 30 +- internal/forwarding/budget.go | 13 +- internal/forwarding/frames.go | 3 - internal/gateway/batch.go | 2 +- internal/gateway/metrics.go | 8 +- internal/gateway/response_budget_test.go | 32 +- internal/gateway/server.go | 104 +--- internal/gateway/stream.go | 21 - internal/metrics/metrics.go | 266 +++------- internal/metrics/metrics_test.go | 17 +- internal/metrics/store_test.go | 5 - internal/metrics/write.go | 2 +- internal/metrics/write_cardinality_test.go | 8 +- internal/protocol/grpc_codec.go | 41 -- internal/protocol/memory.go | 79 --- internal/protocol/memory_test.go | 81 +-- internal/queue/kafka/capacity_split_test.go | 38 +- internal/queue/kafka/worker.go | 13 +- internal/service/admission.go | 475 ------------------ .../service/admission_cancellation_test.go | 85 ---- internal/service/admission_fairness_test.go | 95 ---- internal/service/admission_queue.go | 17 - internal/service/admission_queue_test.go | 179 ------- internal/service/admission_test.go | 146 ------ internal/service/batch_context_test.go | 29 ++ internal/service/batcher.go | 18 - internal/service/batcher_completion_test.go | 2 + internal/service/batching_regressions_test.go | 103 +--- internal/service/batching_server.go | 277 ++-------- .../service/batching_server_internal_test.go | 3 +- internal/service/count_capacity_test.go | 95 ---- internal/service/failure_budget.go | 6 - .../service/failure_budget_internal_test.go | 23 - .../service/lua_compile_cancellation_test.go | 11 +- internal/service/memory_test.go | 170 ------- internal/service/merge_group_test.go | 6 +- internal/service/mutation_batches_test.go | 30 +- internal/service/native.go | 32 -- internal/service/native_boundary_test.go | 162 ++++++ internal/service/native_fixture_test.go | 62 --- internal/service/operation_folding_test.go | 6 +- internal/service/options.go | 148 +----- internal/service/options_test.go | 47 -- internal/service/publish.go | 17 - internal/service/publish_admission_test.go | 179 ------- internal/service/publish_memory_test.go | 94 ---- internal/service/query.go | 17 +- internal/service/read.go | 90 +--- internal/service/read_batching_test.go | 79 +-- internal/service/request_budgets.go | 54 +- .../service/request_budgets_benchmark_test.go | 24 - internal/service/reservation.go | 67 --- internal/service/scan_admission_test.go | 83 --- internal/service/scan_wait_test.go | 239 --------- internal/service/server.go | 38 -- internal/service/server_test.go | 15 +- internal/service/store_capacity_test.go | 44 -- internal/service/sync_capacity_test.go | 37 +- internal/service/working_memory_test.go | 54 ++ internal/service/write.go | 174 +------ internal/service/write_completion_test.go | 6 - internal/service/write_group.go | 49 -- internal/service/write_memory.go | 19 - .../service/write_memory_cancellation_test.go | 48 -- internal/service/write_memory_test.go | 182 ------- internal/service/write_observation_test.go | 3 - internal/service/write_return.go | 4 - internal/storage/mongodb/memory.go | 20 - internal/storage/mongodb/native.go | 24 +- internal/storage/mongodb/query.go | 13 +- internal/storage/mongodb/read.go | 7 - internal/storage/mongodb/scan.go | 8 +- internal/storage/read_budget.go | 98 +--- internal/storage/read_budget_test.go | 42 +- internal/storage/search/client.go | 11 +- .../storage/search/memory_lifetime_test.go | 163 ------ internal/storage/search/native.go | 2 - internal/storage/search/query.go | 7 +- internal/storage/search/read.go | 5 - internal/storage/search/read_budget_test.go | 8 +- internal/storage/search/scan.go | 4 +- internal/storage/search/write.go | 2 - internal/worker/capacity_test.go | 25 +- .../worker/failure_classification_test.go | 51 ++ internal/worker/processor.go | 20 +- proto/forward/forward.proto | 8 +- 138 files changed, 1547 insertions(+), 5936 deletions(-) create mode 100644 internal/app/memory.go create mode 100644 internal/app/memory_test.go create mode 100644 internal/capacity/detect_test.go create mode 100644 internal/capacity/guard.go create mode 100644 internal/capacity/guard_test.go delete mode 100644 internal/capacity/pool.go delete mode 100644 internal/capacity/pool_test.go delete mode 100644 internal/capacity/reader.go delete mode 100644 internal/capacity/scope.go create mode 100644 internal/engine/memory_test.go delete mode 100644 internal/service/admission.go delete mode 100644 internal/service/admission_cancellation_test.go delete mode 100644 internal/service/admission_fairness_test.go delete mode 100644 internal/service/admission_queue.go delete mode 100644 internal/service/admission_queue_test.go delete mode 100644 internal/service/admission_test.go create mode 100644 internal/service/batch_context_test.go delete mode 100644 internal/service/count_capacity_test.go delete mode 100644 internal/service/memory_test.go create mode 100644 internal/service/native_boundary_test.go delete mode 100644 internal/service/native_fixture_test.go delete mode 100644 internal/service/options_test.go delete mode 100644 internal/service/publish_admission_test.go delete mode 100644 internal/service/publish_memory_test.go delete mode 100644 internal/service/reservation.go delete mode 100644 internal/service/scan_admission_test.go delete mode 100644 internal/service/scan_wait_test.go delete mode 100644 internal/service/store_capacity_test.go create mode 100644 internal/service/working_memory_test.go delete mode 100644 internal/service/write_memory.go delete mode 100644 internal/service/write_memory_cancellation_test.go delete mode 100644 internal/service/write_memory_test.go delete mode 100644 internal/storage/mongodb/memory.go delete mode 100644 internal/storage/search/memory_lifetime_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe12769..54fdf67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@4c99c2bfd436db60a2845ed0731b8bf8c5b4557c + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@f9a2683b2946f6f3109c98d2c0a6ef3890a0745a with: - suite_ref: 4c99c2bfd436db60a2845ed0731b8bf8c5b4557c + suite_ref: f9a2683b2946f6f3109c98d2c0a6ef3890a0745a sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/configs/engine.yaml b/configs/engine.yaml index 6ec47c9..b69a6b3 100644 --- a/configs/engine.yaml +++ b/configs/engine.yaml @@ -8,9 +8,9 @@ grpc: max_receive_message_bytes: 64MiB max_send_message_bytes: 64MiB memory: - # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. - burst_percent: 10 # Completion reserve, 1..99. - wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. + # max_bytes: 1GiB # Optional process ceiling; bounded by detected limits. + high_watermark_percent: 80 # Reject new work at this process memory usage. + low_watermark_percent: 70 # Resume new work after usage falls to this level. batching: max_wait: 2ms # Collection delay, not an execution timeout. max_operations: 32 # Measured small-document starting point; not the public RPC limit. @@ -21,11 +21,6 @@ batching: producer: max_buffered_bytes: 64MiB execution: - max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. - max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. - mongodb: # Only valid with the mongodb driver; tune independently in each role. - max_concurrent_writes: 64 - max_concurrent_groups: 16 merge: max_attempts: 3 # Revision-conflict attempts. lua: diff --git a/configs/gateway.yaml b/configs/gateway.yaml index 37cc722..6dc7fd5 100644 --- a/configs/gateway.yaml +++ b/configs/gateway.yaml @@ -8,9 +8,9 @@ grpc: max_receive_message_bytes: 64MiB max_send_message_bytes: 64MiB memory: - # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. - burst_percent: 10 # Completion reserve, 1..99. - wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. + # max_bytes: 1GiB # Optional process ceiling; bounded by detected limits. + high_watermark_percent: 80 # Reject new work at this process memory usage. + low_watermark_percent: 70 # Resume new work after usage falls to this level. request: max_operations: 1000 # Entire client request across all Stores. forwarding: diff --git a/configs/stores/primary.yaml b/configs/stores/primary.yaml index 4ba1630..6dd03be 100644 --- a/configs/stores/primary.yaml +++ b/configs/stores/primary.yaml @@ -7,6 +7,8 @@ storage: uri: mongodb://mongodb:27017 # Example endpoint; required unless uri_file is set. # uri_file: /etc/sink-secrets/mongodb-uri # Omit uri when using its file alternative. metadata_field: __sink # Must match across roles and replicas. + max_concurrent_writes: 64 # Per process; shared by Engine and Worker configuration. + max_concurrent_groups: 16 # For elasticsearch/opensearch, replace mongodb with search: # search: # endpoints: [http://opensearch:9200] diff --git a/configs/worker.yaml b/configs/worker.yaml index daaaa48..33b9018 100644 --- a/configs/worker.yaml +++ b/configs/worker.yaml @@ -3,9 +3,10 @@ # Validate: sink config check --config configs/worker.yaml --store-config configs/stores/primary.yaml mode: worker -# Only the managed capacity is configurable; reservation policy is internal. -# memory: -# max_bytes: 256MiB +memory: + # max_bytes: 1GiB # Optional process ceiling; bounded by detected limits. + high_watermark_percent: 80 + low_watermark_percent: 70 consumer: group_id: primary-workers max_poll_records: 500 @@ -15,11 +16,6 @@ consumer: backoff: 100ms max_backoff: 10s execution: - max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. - max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. - mongodb: # Only valid with the mongodb driver; tune independently in each role. - max_concurrent_writes: 64 - max_concurrent_groups: 16 merge: max_attempts: 3 # Revision-conflict attempts. lua: diff --git a/deploy/reliability-alerts.yaml b/deploy/reliability-alerts.yaml index f097d81..8483119 100644 --- a/deploy/reliability-alerts.yaml +++ b/deploy/reliability-alerts.yaml @@ -31,11 +31,11 @@ groups: annotations: summary: "Sink Kafka fetch errors for store {{ $labels.store }}" description: "Check broker reachability and offset retention gaps; never reset offsets blindly." - - alert: SinkAdmissionSaturated - expr: rate(sink_admission_rejected_total[5m]) > 1 + - alert: SinkMemoryPressure + expr: sink_memory_pressure == 1 for: 5m labels: severity: warning annotations: - summary: "Sink execution capacity is saturated" - description: "Inspect backend latency, input/output reservations and queue limits before adding replicas." + summary: "Sink process memory admission is paused" + description: "Inspect backend latency, process RSS and memory watermarks and queue limits before adding replicas." diff --git a/docs/batching.md b/docs/batching.md index f573263..103f21f 100644 --- a/docs/batching.md +++ b/docs/batching.md @@ -26,7 +26,7 @@ Puts and Merges fold within a group; repeated Reads and Deletes execute once per full address. Executions use their live callers' deadlines and cancellation signals. Write/Delete dispatchers can collect and execute later batches while an earlier -batch waits for refresh. Active batches acquire from the shared memory pool; +batch waits for refresh. New RPCs pass the process memory watermark check; there is no separate request concurrency cap. Record dependencies cover both active and queued RPCs: an RPC touching several records waits for every predecessor, while unrelated RPCs may pass it. Ordering does not @@ -45,9 +45,6 @@ caller touching it has finished its remaining operations for that address, even if their RPCs still contain other unfinished documents. Caller cancellation alone does not release an executing document. Backend bulk calls still return together; Sink cannot acknowledge an item whose backend result is not yet known. -Producer references retain input until execution finishes. Each caller retains -its output through transport completion, so early completion cannot release -still-owned memory. Gateway splits cross-Store requests before forwarding them. Engine accepts only its bound Store and never bypasses that check through the batching layer. Budget-sensitive @@ -62,19 +59,14 @@ omitted. Once a batch is dispatched, other live callers in that batch continue even if one caller cancels. Once all callers cancel, execution is cancelled too. Execution follows the callers' deadlines and cancellation; there is no default whole-request timeout. -Dispatched micro-batches acquire known working allocations from the shared -`memory` pool. Response growth has priority over new arrivals and may borrow its -completion reserve. Request entry fails if ordinary capacity is unavailable; -already admitted queue entries retain their input charge. Async publication -uses the same memory pool; Kafka producer buffers keep their separate bounds. - -Each original RPC retains its snapshot/input/output/returned-document quotas -across chunks, retries and shared-key folding. A full read working set defers -records without consuming caller quota; conditional write chunks retain earlier -successful results. Returned documents acquire capacity before their commit. -Batch splitting uses known request sizes and actual working allocations, rather -than maximum legal responses per caller. Graceful shutdown drains gRPC calls -before stopping batch dispatchers. See [memory admission](design/demand-based-admission.md). +Admitted batches continue through memory pressure. The entire collected batch +executes without independent snapshot/output byte quotas or memory-based splitting. +Record dependencies and adapter/completion grouping still apply. Public response +allowances remain per original RPC; returned documents must fit the gRPC ceiling +before their own commit. Kafka producer buffers and Lua sandbox limits remain. +Queue limits count waiting work only, excluding dispatched batches. Graceful +shutdown drains gRPC calls before stopping batch dispatchers. +See [memory admission](design/demand-based-admission.md). Batching happens only among requests for the same store reaching the same Sink process. More pods increase aggregate queue and storage concurrency, but they @@ -100,11 +92,7 @@ limit and each method's **10,000-operation** waiting queue. A valid explicit RPC with more than 32 operations executes alone; it is not rejected or split into different public requests by this default change. -`memory.burst_percent` remains **10**, the smallest tested completion reserve -that finished every admitted request in the -[allocator saturation experiment](https://github.com/batchstream/sink-production-suite/blob/main/benchmarks/memory-admission/README.md). -The remaining queue, byte, backend-concurrency and Kafka-consumer defaults are -capacity or protection settings without a comparable general tuning result. -Historical workload-specific response caps and GC settings are not carried over -as universal defaults. Tune these separately against actual document sizes, -resource limits and latency targets. +Memory watermarks default to 80% for rejection and 70% for recovery. They are +operational starting points, not benchmark-derived optima. Queue, byte, +backend-concurrency and Kafka-consumer defaults also require workload-specific +validation. Historical response caps and GC settings are not universal defaults. diff --git a/docs/configuration-migration.md b/docs/configuration-migration.md index 14b3302..25e9d41 100644 --- a/docs/configuration-migration.md +++ b/docs/configuration-migration.md @@ -11,13 +11,16 @@ from an incompatible forwarding version; old and new Engines cannot share routes storage settings from their component files. Gateway takes only `--config`. 3. Move `gateway` to `forwarding`. Keep only `request.max_operations` on Gateway. 4. Move `service.batching` to Engine's `batching`; move `service.merge` to - `execution.merge`. Put MongoDB concurrency under `execution.mongodb`. + `execution.merge`. Put MongoDB concurrency in the shared Store under `storage.mongodb`. 5. Move Kafka `producer` to Engine and `consumer` to Worker. Worker has no `grpc`, - `request`, `batching`, or `producer`, and `memory` accepts only `max_bytes`. + `request`, `batching`, or `producer`, and uses the same memory watermarks as other roles. 6. Remove `service.request.timeout` and response/read quotas. Caller contexts control request lifetime; gRPC message limits bound transport responses. - Snapshot/output working sets use `execution.max_snapshot_bytes` and - `execution.max_output_bytes`. Remove old count-based execution/publish admission. + Remove snapshot/output byte quotas and old count-based execution/publish admission. + Process watermarks now control new admission. Remove `memory.burst_percent` and + `memory.wait_timeout`; use `high_watermark_percent` (80) and + `low_watermark_percent` (70). Startup panics if estimated minimum working memory + cannot fit below the high watermark. See the [sizing formula](design/demand-based-admission.md). 7. Move Kafka `topic.partitions`, `topic.replication_factor`, `topic.min_insync_replicas`, and `topic.max_record_bytes` directly under `kafka`; they apply to both Topics. Rename `dead_letter.topic` to `dead_letter.name`. diff --git a/docs/configuration.md b/docs/configuration.md index 542a2bd..aa63b1b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,12 +34,12 @@ startup. Restart after changes. Configuration files are limited to 4 MiB. | --- | --- | | Role: `mode`, `shutdown_timeout` | Process role and graceful shutdown | | Role: `grpc`, `health`, `prometheus`, `logging` | Listeners and observability; Worker has no `grpc` | -| Role: `memory` | Process capacity; Worker accepts only optional `max_bytes` | +| Role: `memory` | Process memory ceiling and admission watermarks | | Gateway: `request.max_operations` | One public batch across all Stores | | Gateway: `forwarding` | Store routes, discovery and fanout | | Engine: `batching`, `producer` | Collection queues and Kafka publishing buffer | | Worker: `consumer` | Group, polling, processing rounds and retries | -| Engine/Worker: `execution` | Snapshot/output working sets, MongoDB concurrency and Lua | +| Engine/Worker: `execution` | Merge conflict retries and Lua sandbox | | Store: `name`, `storage`, `kafka` | Shared identity, database connection and Topic/DLQ policy | Request validation belongs to Gateway. Engine has no `request` section and accepts @@ -77,15 +77,18 @@ storage: mongodb: uri: mongodb://mongodb:27017 metadata_field: __sink + max_concurrent_writes: 64 + max_concurrent_groups: 16 kafka: enabled: true brokers: [kafka:9092] + partitions: 4 + replication_factor: 2 + min_insync_replicas: 1 topic: name: catalog-mutations - partitions: 4 - replication_factor: 2 dead_letter: - topic: catalog-mutations.dlq + name: catalog-mutations.dlq ``` Kafka is optional for Engine. Without it, synchronous calls remain available and @@ -141,8 +144,8 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. | `storage.driver` | enum string | Yes | none | `mongodb`, `elasticsearch`, `opensearch` | Adapter used by this storage instance. See [Storage driver values](#storage-driver-values). | | `storage.mongodb.uri` | string | Conditionally | none | Valid MongoDB connection string | Required when the entry's driver is `mongodb`. | | `storage.mongodb.metadata_field` | string | No | `__sink` | Any valid MongoDB field except `_id`; cannot contain `.`, `$`, or a null byte | Reserved top-level field where Sink stores internal metadata such as the record revision; removed from documents returned to clients. | -| `execution.mongodb.max_concurrent_writes` | positive integer | No | `64` | Integer greater than `0` | Maximum concurrent MongoDB conditional writes. | -| `execution.mongodb.max_concurrent_groups` | positive integer | No | `16` | Integer greater than `0` | Maximum collection groups executed concurrently across all calls to one store. | +| `storage.mongodb.max_concurrent_writes` | positive integer | No | `64` | Integer greater than `0` | Maximum concurrent MongoDB conditional writes. | +| `storage.mongodb.max_concurrent_groups` | positive integer | No | `16` | Integer greater than `0` | Maximum collection groups executed concurrently across all calls to one store. | | `storage.search.endpoints` | list of strings | Conditionally | none | One or more HTTP(S) endpoints | Required for `elasticsearch` and `opensearch`. | | `storage.search.username` | string | Conditionally | empty | Any username accepted by the search service | Basic-auth username. Must be configured together with `password`. | | `storage.search.password` | string | Conditionally | empty | Any password accepted by the search service | Basic-auth password. Must be configured together with `username`. | @@ -178,7 +181,7 @@ use the lowercase spelling shown below. Storage names are also case-sensitive. There is no server-wide business request timeout and no default SDK Scan deadline. Callers choose deadlines or wait until completion, cancellation or process shutdown. -`memory.wait_timeout` bounds capacity contention, `execution.merge.lua.timeout` +`execution.merge.lua.timeout` bounds a Lua invocation, and `consumer.processing_timeout` bounds a Worker processing round; none of these is a whole-RPC deadline. @@ -186,23 +189,17 @@ There is no `request.max_response_bytes` or `request.max_read_bytes` knob. Gatew uses `grpc.max_send_message_bytes` as the transport ceiling and reserves encoded result overhead before forwarding. Returned documents share the remaining budget across all Store groups of the original RPC. Oversized returned writes are rejected -before their own commit. Execution snapshots and candidate outputs have independent -Store-local working sets; they do not consume the public response allowance. - -| Execution setting | Default | Meaning | -| --- | --- | --- | -| `execution.max_snapshot_bytes` | `32MiB` | Conditional write snapshot working set | -| `execution.max_output_bytes` | `32MiB` | Candidate write output working set | - +before their own commit. Intermediate snapshots and candidate outputs use process +memory protection without independent quotas. Collected batches execute together. All settings in this table are optional. Counts and bytes are positive integers; time values are positive duration strings such as `30s` or `2ms`. Limits are process-local; replica counts multiply capacity. Configure the same Kafka policy on servers and workers. | Setting | Default | Meaning | | --- | --- | --- | -| `memory.max_bytes` | automatic | Shared managed capacity for this process; omit to use half the smallest detected Go/cgroup/host memory limit, or 256 MiB fallback. Explicit values must be at least 1 KiB. Not an RSS cap. | -| `memory.burst_percent` | `10` | Completion reserve as a percentage of total capacity, from 1 to 99. New requests cannot enter through this reserve. | -| `memory.wait_timeout` | `2s` | Maximum response/working-set growth wait. Caller deadlines take precedence. New requests fail immediately when ordinary capacity is unavailable. | +| `memory.max_bytes` | automatic | Optional process ceiling, at least 1 KiB; clamped to detected Go/cgroup/host limits. Fallback is 1 GiB. Startup also checks the minimum required working memory. | +| `memory.high_watermark_percent` | `80` | Reject new requests or pause Worker polling at this percentage of the effective ceiling; integer from 1 to 99. | +| `memory.low_watermark_percent` | `70` | Resume admission at or below this percentage; positive integer strictly below the high watermark. | | `kafka.dead_letter.retention` | `720h` | Independent DLQ retention, 30 days; at least `1ms` and bounded by Go duration range. | | `kafka.min_insync_replicas` | `1` | Minimum ISR for both Topics, at most replication factor. Publishers require all current ISR acknowledgements; the default permits writes with one in-sync replica during broker maintenance. | | `kafka.max_record_bytes` | `900KiB` | Encoded mutation envelope plus key, including expanded Lua source; at most 64 MiB; Engine producer buffer must cover it. Topic/producer batch limits include an extra 16 KiB for framing and DLQ headers. Broker/replica fetch limits must also support increases. | @@ -214,32 +211,21 @@ counts multiply capacity. Configure the same Kafka policy on servers and workers reconciles this shared policy rather than relying on broker defaults. `kafka.topic` and `kafka.dead_letter` each contain only `name` and `retention`, which are independent. -All request classes share the same process pool. Input, small completion -envelopes and forwarding scratch space acquire ordinary capacity. Document -copies and response encoding acquire capacity using known sizes; increasing the -legal maximum response does not increase the price of every small request. -Responses have priority over new arrivals. A selected completion owner may use -the reserve; waits remain bounded even when a response cannot make progress. -There is no separate request concurrency cap in this version. - -The default reserve was chosen from the [reproducible saturation experiment](https://github.com/batchstream/sink-production-suite/blob/main/benchmarks/memory-admission/README.md). -Override it for your measured workload. Automatic capacity detection runs once -at startup, leaving half the effective limit for unmanaged runtime/driver/GC -memory. The source is exported in `sink_memory_capacity_source_info`. - -MongoDB reads and native commands keep a temporary 48 MiB driver wire allowance -only around database work; actual document copies are accounted separately. -Search bodies acquire actual backing-array capacity as they grow. Lua VM heaps, -Kafka buffers, ingress decoding and GC still require process headroom. -Returned-document capacity is acquired before the corresponding commit. Scope -ownership follows batching and gRPC buffers beyond handler return and cancellation. -See [memory admission](design/demand-based-admission.md) and -[metrics/KEDA migration](observability.md#memory-capacity-and-keda). - -The former `service` sections and Gateway count/byte admission settings are -rejected. Use `memory` for process capacity and `batching.queue` for Engine queues. -Gateway/Engine expose `burst_percent` and `wait_timeout`; Worker uses internal -completion-reserve and wait defaults and exposes only optional `max_bytes`. +All roles use the same watermark settings. Linux measures process RSS; other +platforms fall back to Go runtime memory. Sampling is cached for 100 ms. Existing +requests continue after the high watermark is reached; Worker finishes its current +poll before pausing. There is no separate request concurrency cap. + +Before opening dependencies, Sink estimates minimum working memory from transport, +Lua, driver and Kafka settings. Startup panics if the high-watermark portion of the +effective ceiling cannot cover it. The panic lists the required components and +available ceiling. This catches undersized configurations; it does not guarantee +against OOM under concurrent load. See the [sizing formula and runtime behavior](design/demand-based-admission.md) +and [metrics/KEDA migration](observability.md#memory-capacity-and-keda). + +The former `service` sections, snapshot/output quotas, `memory.burst_percent`, +`memory.wait_timeout` and Gateway count/byte admission fields are rejected. +Use `memory` for process pressure and `batching.queue` for Engine waiting queues. MongoDB group concurrency is shared across concurrent calls. Sink sets `w=majority` and `journal=true` on its client, overriding weaker URI concerns; diff --git a/docs/design/demand-based-admission.md b/docs/design/demand-based-admission.md index 7d77a1e..7eddeab 100644 --- a/docs/design/demand-based-admission.md +++ b/docs/design/demand-based-admission.md @@ -1,104 +1,86 @@ -# Demand-based memory admission - -The application uses one process-local capacity pool for Gateway, Engine or -Worker. Logical per-RPC document limits remain independent of allocation -ownership: increasing `grpc.max_send_message_bytes` no longer prepays that -maximum for every ordinary request. - -## Capacity and admission - -`memory.max_bytes` overrides automatic sizing. Otherwise Sink takes half of the -smallest available Go memory limit, cgroup v1/v2 ancestor limit, or Linux host -memory total. An environment without a discoverable limit uses 256 MiB. Detection -runs at startup; changing container limits requires restarting the process. -The other half provides headroom for runtime metadata, ingress decoding, Kafka, -Lua heaps, caches and GC. This is a managed capacity limit, **not an RSS cap**. - -`memory.burst_percent` defaults to 10, leaving 90% for ordinary allocations. -The default is the smallest tested reserve that completed every accepted request -in the saturation experiment; see [the experiment](https://github.com/batchstream/sink-production-suite/blob/main/benchmarks/memory-admission/README.md). -It is configurable from 1 to 99 and is not a universal optimum. - -New requests acquire known input, small completion envelopes and forwarding -scratch space from ordinary capacity. They fail immediately when unavailable, -so decoded requests cannot accumulate in an unaccounted admission queue. -Already admitted requests in the batching queue keep their input ownership. -There is no request concurrency cap in this policy. Existing backend connection, -fanout, batch queue and Kafka buffer bounds still apply. - -Response/working-set growth has priority over new arrivals. Growth acquires the -whole next allocation atomically; only one completion owner can borrow the -reserve at a time. Coalesced work is one owner, but each original RPC retains and -releases its own output leases. This prevents a batch waiting on its own reserve -while preserving original caller ownership. - -Response waits end at `memory.wait_timeout` (default 2s) or the caller deadline, -whichever comes first. At most 1024 growth acquisitions can wait. An allocation -that cannot fit even after every other owner releases fails immediately. A -reserve smaller than the next allocation cannot guarantee progress when other -requests retain all ordinary space; bounded failure is intentional. Oversized -responses and sustained saturation still require smaller requests or capacity. - -## What is charged - -- Generated `SizeVT()` measures protobuf wire size, not Go heap size. Input - accounting adds object/envelope allowances. Known returned documents reserve - their retained payload and eventual encoded copy together, before commit. - Shared Lua declarations remain shared during parsing; asynchronous writes - reserve their expanded queue envelopes before serialization and enqueueing. -- Read snapshots acquire capacity before adapter copies. Original caller outputs - acquire separately before response copies. Logical snapshot/input/output/ - returned-document quotas still belong to the original RPC across retries, - batching and Gateway fanout. -- Search HTTP bodies acquire backing-array capacity before growth, including the - overlap of old/new arrays and JSON decoding space. Lua VM allocation is not - controlled by protobuf size; existing Lua limits and process headroom remain - necessary. Discarded HTTP bodies and rejected candidates release capacity - immediately; retained decoded documents keep their own reservations. - Candidates acquire capacity before retention and commit. -- MongoDB hides complete wire allocation inside its driver. Reads and native - operations reserve a temporary 48 MiB allowance around database work. It is - included in used bytes and reported separately as opaque reservation. It is - not charged at RPC entry and does not scale with the configured response limit. - Count's internal cursor pages release working-set capacity after consumption. -- Small completion envelopes are acquired before execution. Failure to acquire a - returned document fails that operation before its write; completed operations - keep their result. Transport failure after a write remains an ambiguous reply, - and never authorizes replaying a mutation. - -## Ownership and transport - -RPC scopes are reference counted. Cancellation does not free bytes while a -batch producer still owns its input. Service completion does not release a -response's encoding allocation while HTTP/2 still holds buffer references. -The VT codec attaches ownership to the final `grpc/mem.Buffer` release. Tiny -messages use an allocation above grpc's small-buffer threshold, since -`SliceBuffer.Free` otherwise provides no release callback. - -Public Sink RPCs and generated public messages are unchanged. The private Engine -adds `ForwardStream`: first an encoded-size frame, then chunks of at most 32 KiB. -Gateway acquires the announced actual size before continuing receipt. Its fixed -frame receive limit also bounds a dishonest peer; the announced size cannot -raise that limit. Static 64 KiB stream and 1 MiB connection windows avoid -unbounded response prefetch while capacity is unavailable. - -Upgrade Engines before Gateways. Old Gateways can continue calling the retained -unary `Forward` method. A new Gateway does not fall back to unbounded unary -receiving or replay an uncertain request when `ForwardStream` is unavailable. -Rollback Gateways before Engines. See [rolling upgrades](../rolling-upgrades.md). - -## Observability and compatibility - -The `sink_memory_*` family exposes ordinary/reserved capacity, owned bytes, -opaque driver reservations, pending growth bytes/count/age, reserve use, -acquisitions, rejections and wait duration, with bounded `role`/`store` labels. -The KEDA example uses fleet pressure with `metricType: Value`, plus a rejection -ratio to capture bursts that disappear between scrapes. Missing telemetry is an -error, not zero load. Keep synchronous replica floors above zero and monitor -process RSS separately. See [observability](../observability.md#memory-capacity-and-keda). - -The CLI always enables this policy. Older `gateway.max_bytes`, Gateway request -count fields, `service.execution` and `service.publish` admission fields were previously -accepted for compatibility; the role-config schema now rejects them. Application assembly always supplies the shared pool. -Batch queue limits, request/response size limits, deadlines and backend resource -limits remain active. Migrate autoscaling away from the old reservation gauges. +# Process memory admission + +Sink uses one high/low watermark guard per process. It does not track individual +allocations, ownership, reservations, or separate snapshot/output quotas. + +## Startup sizing + +At startup, the effective ceiling is the smallest detected finite Go memory +limit (`GOMEMLIMIT`), cgroup v1/v2 limit (including ancestors), or host memory +size. If none is available, the fallback is 1 GiB. `memory.max_bytes` may lower +that ceiling; on the fallback path it supplies the ceiling directly. Detection +runs once, so resource-limit changes require a restart. + +Before opening dependencies or listeners, Sink estimates minimum working memory: + +| Component | Estimate | Roles | +| --- | --- | --- | +| Runtime and drivers | 64 MiB | All | +| Request decoding | 2 × `grpc.max_receive_message_bytes` | Gateway, Engine | +| Response encoding | 2 × `grpc.max_send_message_bytes` | Gateway, Engine | +| Merge documents | 2 × `execution.merge.lua.max_result_bytes` | Engine, Worker | +| Lua source cache | `max_source_bytes` × `max_cached_programs` | Engine, Worker | +| MongoDB wire buffers | 48 MiB | MongoDB Engine, Worker | +| Kafka producer buffer | `producer.max_buffered_bytes` | Engine with Kafka | +| Kafka fetch and decoded batch | 48 MiB | Worker | +| Dead-letter producer buffer | max(64 MiB, `kafka.max_record_bytes` + 16 KiB) | Worker | + +The sum must fit **below or at the high watermark**. Otherwise startup panics +with role, effective ceiling and source, high watermark, required bytes, and the +component breakdown. `sink config check` validates files only; it deliberately +does not validate the checking machine's available resources. + +With default settings, minimum working memory is 320 MiB for Gateway, +416 MiB for a MongoDB Engine without Kafka, 480 MiB for one with Kafka, and +272 MiB for a MongoDB Worker. At the default 80% high watermark, the effective +ceilings must be at least 400, 520, 600, and 340 MiB respectively. Search omits +the 48 MiB MongoDB allowance. Overrides change these estimates. + +These are sizing estimates for a normal work unit, not reservations or a promise +that every concurrent workload fits. Compiled Lua programs, expanded documents, +driver pools and simultaneous work can use more memory. The estimate intentionally +does not multiply every legal maximum by unbounded concurrency or by queue size. + +## Runtime behavior + +`memory.high_watermark_percent` defaults to 80 and +`memory.low_watermark_percent` to 70, with `0 < low < high < 100`. +Linux measures resident process memory through `/proc/self/statm`. When that is +unavailable or invalid, Sink uses Go runtime memory (`Sys - HeapReleased`). The +metric's `source` label distinguishes these measurements. The latter does not +include all native allocations. + +Admission checks and metric scrapes sample at most once per 100 ms. Reaching the +high watermark stops new admission. Admission resumes only at or below the low +watermark, avoiding rapid oscillation around one threshold. + +- Gateway rejects a new business RPC with `RESOURCE_EXHAUSTED` before forwarding. +- Engine checks its own process pressure before execution. Private responses mark + this rejection `not_started`, so Gateway preserves safe retry semantics. +- Worker pauses fetching before the next poll and resumes below the low watermark. + Already polled work completes its normal processing and settlement. Pausing does + not acknowledge records or advance offsets; group heartbeats remain active. +- Admitted work continues, including collected batches, conflict retries and + response encoding. Crossing a watermark never cancels that work. + +The request body is already decoded when admission runs. Concurrent arrivals +between samples, admitted work, Kafka prefetch, the runtime, and drivers can still +raise memory above the ceiling. This is overload control, **not an OOM guarantee**. +GC reclamation and RSS release are asynchronous; memory may remain high after a +request finishes. No allocation-level counters or forced GC loops are used. + +## Batch and response boundaries + +Engine processes the collected batch without further snapshot/output memory +splitting. Record ordering, adapter grouping, completion modes and conflict +retries still determine execution groups. The waiting queue retains its encoded +byte and operation limits; dispatched work no longer counts toward those limits. + +The gRPC message ceiling still limits public responses. Each original RPC keeps +its response allowance across Stores, batching and retries. A requested returned +document must fit before its own write commits. This is a wire-format limit, +separate from process memory protection. Lua sandbox and Kafka buffer limits also +remain in effect. + +See [configuration](../configuration.md#reliability-limits) and +[memory metrics and KEDA](../observability.md#memory-capacity-and-keda). diff --git a/docs/native-access.md b/docs/native-access.md index 62a5fe1..bbe36f7 100644 --- a/docs/native-access.md +++ b/docs/native-access.md @@ -431,30 +431,26 @@ from changing the record. ## Limits, deadlines, and retries -Native RPCs use the shared process memory pool and honor the caller context. +Native RPCs use the process memory watermark guard and honor the caller context. `ClientOptions.ScanTimeout` is opt-in; zero adds no deadline. Between Scan calls there is no held request capacity or background cursor. -New requests acquire ordinary memory capacity or fail immediately. Temporary +New requests fail immediately while process memory pressure is active. Temporary refusal before backend execution preserves Scan's safe retry detail: `google.rpc.ErrorInfo` with `domain="sink"`, `reason="SCAN_ADMISSION_REJECTED"` and metadata `pool=memory`, `reason=busy`. Oversized requests and failures after -execution begins do not advertise this pre-execution guarantee. Actual response -growth may wait up to `memory.wait_timeout` (default 2s), bounded by the page's -original deadline. Cancellation and deadline expiry retain `CANCELED` and +execution begins do not advertise this pre-execution guarantee. Admitted work follows the caller deadline and continues through memory pressure. Cancellation and deadline expiry retain `CANCELED` and `DEADLINE_EXCEEDED` respectively. See [memory admission](design/demand-based-admission.md). Execute responses and returned Write documents share the gRPC send ceiling semantics; returned-document budgets are per original RPC even after batching. Count uses a separate backend response budget of min(the gRPC send ceiling, -256 KiB), enforced by the adapter as well as admission. MongoDB counts still -reserve the driver's 48 MiB wire ceiling. This keeps small count responses from -reserving a full document page. +256 KiB), enforced by the adapter. This limits backend page sizes without per-allocation memory accounting. Output space is reserved before committing a returned write. A candidate that cannot fit fails before its own write; earlier operations may already be applied. Scan pages use at most min(the gRPC send ceiling, 4 MiB), with count and byte limits both enforced. A single oversized document fails with -`RESOURCE_EXHAUSTED`. Search grows its backend response buffer against actual capacity, within the +`RESOURCE_EXHAUSTED`. Search bounds its backend response buffer within the logical limit of two page budgets plus 64 KiB of metadata and the store response limit. It reduces the hit count when that buffer is exceeded, preserving one lookahead hit and validating the complete response before returning a smaller page. A stricter store response diff --git a/docs/observability.md b/docs/observability.md index 06b8795..56f755f 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -41,7 +41,7 @@ Sink metrics: | `sink_batcher_queued_operations` | gauge | `store`, `method` | Operations currently waiting for dispatch. | | `sink_batcher_queued_bytes` | gauge | `store`, `method` | Encoded request bytes currently waiting for dispatch. | | `sink_batcher_rejected_total` | counter | `store`, `method`, `reason` | Requests rejected before dispatch, including queue exhaustion. | -| `sink_write_phase_duration_seconds` | histogram | `store`, `phase` | Synchronous core write admission, parsing, storage read, Lua, and storage write latency by store; writes distinguish applied/visible. | +| `sink_write_phase_duration_seconds` | histogram | `store`, `phase` | Synchronous core write parsing, storage read, Lua, and storage write latency by store; writes distinguish applied/visible. | | `sink_write_execution_rounds` | histogram | `store`, `phase` | Actual storage read/write calls per synchronous core write, including conflict retries, by store. | | `sink_write_slow_phases_total` | counter | `store`, `phase` | Phase observations exceeding 5 seconds, by store and phase. | | `sink_merge_conflicts_total` | counter | `store` | Revision conflicts retried by Lua merge operations. | @@ -53,19 +53,7 @@ Sink metrics: | `sink_kafka_worker_mutations_total` | counter | `store`, `status` | Mutations applied or failed by workers. | | `sink_kafka_worker_retries_total` | counter | `store` | Retried Kafka mutations. | | `sink_kafka_worker_dead_letters_total` | counter | `store` | Mutations copied to the dead-letter topic. | -| `sink_in_flight_requests` | gauge | none | Executing core calls in this process. | -| `sink_in_flight_bytes` | gauge | none | Request/output reservations; not RSS. | -| `sink_admission_rejected_total` | counter | none | Process execution admission rejections. | -| `sink_admission_pool_requests` | gauge | `store`, `pool` | Executing requests in the independent `execution` or `publish` pool. | -| `sink_admission_pool_bytes` | gauge | `store`, `pool` | Bytes reserved in each independent pool. The legacy in-flight gauges report their sum. | -| `sink_admission_pool_rejected_total` | counter | `store`, `pool`, `reason` | Rejections from request/scan slots (`requests`), global bytes (`bytes`), an older byte waiter (`fairness`), a full direct/Scan waiting queue (`queue`), admission wait expiry (`wait_timeout`), or write reservation growth (`resize`). | -| `sink_execution_queued_requests` | gauge | `store` | Direct synchronous RPCs waiting for admission, separate from batching and Scan queues. | -| `sink_execution_queued_bytes` | gauge | `store` | Input and bookkeeping bytes charged to the direct admission queue. | -| `sink_execution_admission_wait_duration_seconds` | histogram | `store` | Direct RPC queue time until admission, rejection or cancellation. | -| `sink_scan_queued_requests` | gauge | `store` | Scan pages waiting for execution admission. | -| `sink_scan_queued_bytes` | gauge | `store` | Conservative reservation bytes charged to the separate Scan waiting queue. | -| `sink_scan_admission_wait_duration_seconds` | histogram | `store` | Time queued Scan pages waited before admission, rejection or cancellation. | -| `sink_execution_store_bytes` | gauge | `store` | Execution bytes reserved by this process, attributed to its bound Store. | +| `sink_in_flight_requests` | gauge | none | Admitted Engine RPCs still queued or executing. | | `sink_kafka_worker_last_poll_timestamp_seconds` | gauge | `store` | Last completed poll, not an idle-worker heartbeat. | | `sink_kafka_worker_last_commit_timestamp_seconds` | gauge | `store` | Last successful offset commit. | | `sink_kafka_worker_pending_records` | gauge | `store` | Unresolved records from the last fetch; excludes unpolled backlog. | @@ -82,7 +70,6 @@ creating arbitrary time series. Gateway forwarding metrics use the configured route Store names; the Gateway splits cross-Store RPCs before Engine execution. Kafka observations use the bound Store, including malformed or misrouted messages sent to that Worker's DLQ. Admission pool metrics attribute reservations to the -bound Store. The `sink_in_flight_*` and `sink_admission_rejected_total` metrics remain process totals across pools. Build info and standard Go/process collectors also remain process-wide. @@ -123,7 +110,7 @@ outcome or completion mode; it supports per-store distributions. The legacy queue histogram still observes only the oldest request in each dispatched batch. The write phase histogram uses the same seven finite latency buckets and only -six phase values: `admission`, `parse`, `storage_read`, `lua`, +five phase values: `parse`, `storage_read`, `lua`, `storage_write_applied`, and `storage_write_visible`. Non-write phases combine applied and visible modes. Async acceptance uses the existing Kafka metrics and is excluded from these synchronous write diagnostics. Overall latency remains @@ -149,12 +136,12 @@ writes, round counts, conflict counters, and backend statistics before attributing a slow batch to refresh. Several short phases or retries can also produce a slow overall RPC without incrementing the slow-phase counter. -The series budget for these write diagnostics is **123 × S + 168 per Pod**, where +The series budget for these write diagnostics is **112 × S + 146 per Pod**, where S is the configured store count (one per Engine or Worker). This includes both fixed fallback labels for write phases/rounds, all possible phase/outcome combinations, `+Inf`, `_sum`, and `_count`. The breakdown is 30 queue-histogram series and 9 queue-exit counters per configured -store, plus 60 phase-histogram series, 18 round-histogram series, and 6 slow-phase -counters per store/fallback. Queues exist only for configured stores. For six single-Store Engine Pods, the upper bound is **1,746 series** for these diagnostics. +store, plus 50 phase-histogram series, 18 round-histogram series, and 5 slow-phase +counters per store/fallback. Queues exist only for configured stores. For six single-Store Engine Pods, the upper bound is **1,548 series** for these diagnostics. Series are created on observation. Other metrics and historical Pod churn are outside this budget. A regression test exports both Prometheus text and OpenMetrics with 1, 3, and 16 label values to enforce the collector budget; it also checks that @@ -176,49 +163,30 @@ See the [configuration reference](configuration.md) for listener settings. ## Memory capacity and KEDA -The CLI's shared allocator replaces speculative execution/forwarding admission. -Use these metrics for the new policy; legacy execution/Scan/publishing and -Gateway reservation gauges describe the old allocator and must not drive new -capacity decisions. Existing RPC, backend, Kafka and batch queue metrics remain -useful. Every memory series has bounded `role` and `store` labels (Gateway's -store is empty). +Each process exports its watermark guard with bounded `role` and `store` labels +(Gateway's Store label is empty). Allocation/lease, burst reserve, opaque driver +allowance and waiting-allocation metrics have been removed. | Metric | Type | Additional labels / meaning | | --- | --- | --- | -| `sink_memory_capacity_bytes` | gauge | `pool=normal,burst`; sum is configured or detected managed capacity | -| `sink_memory_used_bytes` | gauge | `pool=normal,burst`; held input, result, copies and driver capacity, not RSS | -| `sink_memory_opaque_reserved_bytes` | gauge | Temporary driver allowance included in used bytes; do not add it again | -| `sink_memory_waiting_bytes` | gauge | `phase=request,response`; additional requested allocation bytes | -| `sink_memory_waiting_requests` | gauge | `phase`; waiting acquisitions, not a concurrency limit | -| `sink_memory_oldest_wait_seconds` | gauge | `phase`; zero when idle | -| `sink_memory_burst_borrowers` | gauge | Zero or one completion owner | -| `sink_memory_admitted_total` | counter | `phase`; successful acquisitions | -| `sink_memory_rejected_total` | counter | `phase`, `reason=busy,oversize,wait_timeout` | -| `sink_memory_wait_seconds` | histogram | `phase`, `outcome=admitted,canceled`; timed-out waits count as canceled | -| `sink_memory_capacity_source_info` | gauge | `source=configured,gomemlimit,cgroup,host,fallback` | - -Waiting and rejection series include idle zeros. Request admission fails -immediately if full, so request wait gauges normally stay zero. Response wait -bytes represent outstanding demand; retries can contribute repeated acquisition -attempts to counters. Oversized allocations are permanent capacity failures and -should not trigger autoscaling through a rejection-rate metric. - -The [KEDA example](../examples/autoscaling/memory-keda.yaml) uses two signals: -held plus pending bytes divided by ordinary fleet capacity, and the recent -fraction of temporary request refusals. Both use `metricType: Value` because they -are fleet ratios. Scope every selector to exactly one Deployment, role and, for -Engines/Workers, Store. Do not combine Gateway and Engine capacity or use raw -monotonic counters as scaling values. Keep Worker Kafka lag triggers as well. - -Replace example job/namespace names with the actual scrape labels. The capacity -query rejects incomplete scrapes rather than interpreting missing capacity as -idle; `ignoreNullValues: "false"` makes missing data a scaler error. Maintain -nonzero synchronous replica floors and monitor Prometheus target health. -See KEDA's [Prometheus scaler](https://keda.sh/docs/2.18/scalers/prometheus/) and -[metric types](https://keda.sh/docs/2.18/reference/scaledobject-spec/). - -Monitor `process_resident_memory_bytes` and Go heap metrics alongside these -series. Managed ownership is not GC reclamation or RSS: ingress decoding, -Kafka buffering, Lua and runtime objects require the automatic sizing headroom. -A high opaque-reservation share identifies MongoDB wire protection rather than -large logical response quotas. +| `sink_memory_limit_bytes` | gauge | Effective process ceiling | +| `sink_memory_used_bytes` | gauge | `source=rss,go_runtime`; observed process usage | +| `sink_memory_watermark_bytes` | gauge | `watermark=high,low`; reject and recovery thresholds | +| `sink_memory_pressure` | gauge | 1 while new work is blocked; otherwise 0 | +| `sink_memory_admitted_total` | counter | Requests admitted at Gateway/Engine entry | +| `sink_memory_rejected_total` | counter | Requests rejected before execution | +| `sink_memory_limit_source_info` | gauge | `source=configured,gomemlimit,cgroup,host,fallback` | + +Worker uses the pressure gauge and Kafka lag; its poll pause is not a rejected RPC. +Usage is sampled at most once per 100 ms. It includes resident memory retained by +the runtime and does not return to zero when requests finish. The fallback Go +runtime measurement excludes some native allocations. See [memory policy](design/demand-based-admission.md). + +The [KEDA example](../examples/autoscaling/memory-keda.yaml) uses observed usage +divided by the fleet's high watermarks, plus the recent fraction of refused RPCs. +Both use `metricType: Value` for fleet ratios. Scope selectors to one Deployment, +role and Store. Keep Worker Kafka lag triggers. Missing scrapes are scaler errors, +not zero pressure; retain nonzero replica floors and monitor target health. + +Monitor `process_resident_memory_bytes` and Go heap metrics alongside these signals. +Watermarks reduce overload but do not reserve memory for already admitted work. diff --git a/docs/production-sizing.md b/docs/production-sizing.md index 1f39745..75aafea 100644 --- a/docs/production-sizing.md +++ b/docs/production-sizing.md @@ -10,19 +10,18 @@ capacity measurements with the [rollout drain budget](rolling-upgrades.md). ## Process budgets -All roles use the `memory` pool: automatic detection leaves half the effective -runtime/container limit for non-managed memory, with 10% of managed capacity -reserved for completion. Override `memory.max_bytes` and `memory.burst_percent` -only with workload measurements. No request execution concurrency cap is added. -Batch queues, Gateway fanout/connections, backend pools, Kafka buffers and Lua -limits remain separately bounded. See the [reserve experiment](https://github.com/batchstream/sink-production-suite/blob/main/benchmarks/memory-admission/README.md) +All roles use process memory watermarks: reject/pause at 80% and resume at 70% +of the effective Go/container/host ceiling. Startup panics if estimated minimum +working memory exceeds the high-watermark allowance. See the [sizing formula](design/demand-based-admission.md) and [memory/KEDA signals](observability.md#memory-capacity-and-keda). +This is overload control, not an OOM guarantee. There is no separate execution +concurrency cap. Batch queues, Gateway fanout/connections, backend pools, Kafka +buffers and Lua limits remain separately bounded. A 2 CPU / 4 GiB allocation alone does not determine safe RPC concurrency. A small Put, a returned-document Merge, and a BSON Scan have different memory and CPU costs. Size against the actual request mix and backend latency. Leave memory for -transport, Kafka buffers, Lua VMs, driver buffers and garbage collection outside -configured document budgets. Configure Go memory settings within the container +transport, Kafka buffers, Lua VMs, driver buffers and garbage collection within the process budget. Configure Go memory settings within the container limit and measure rather than reusing settings from old architecture benchmarks. ## Database connection budget @@ -40,8 +39,7 @@ can affect multiple Stores, so it needs independent capacity planning. Each search Store owns its HTTP connection pool and retains at most 128 idle connections across all endpoints, with a 90-second idle timeout. This avoids -reopening most connections after concurrent bursts. Active requests acquire -managed memory capacity; 128 is an idle-cache limit, not a limit on active +reopening most connections after concurrent bursts. Active requests pass process memory admission; 128 is an idle-cache limit, not a limit on active connections. Shutdown releases the Store's idle connections after work drains. ## Scaling signals diff --git a/docs/rolling-upgrades.md b/docs/rolling-upgrades.md index 4c7126c..2cf9714 100644 --- a/docs/rolling-upgrades.md +++ b/docs/rolling-upgrades.md @@ -12,13 +12,15 @@ already accepted request. ## Memory-admission protocol upgrade -Upgrade all Engines before Gateways when introducing demand-based memory -admission. New Gateways call the private `ForwardStream` method; old Gateways -remain compatible with the retained unary `Forward` method on new Engines. -There is no fallback that replays a mutation or receives an unreserved maximum -unary response. Roll back Gateways before Engines. Public Sink RPCs and SDK -messages do not change. Migrate autoscaling to the `sink_memory_*` metrics before -using the new policy; legacy admission settings no longer control CLI capacity. +The new private protocol is version 6. It retains only response allowances and +removes snapshot/input/output grants. Engines reject other protocol versions; +retaining both `Forward` and `ForwardStream` methods does not provide compatibility +with older versions. Use matching Gateway/Engine deployments and an isolated +cutover for incompatible versions. Public SDK messages remain unchanged. + +Replace legacy admission/reservation dashboards and alerts with the new +[memory watermark metrics](observability.md#memory-capacity-and-keda). Old metric +families are removed, not kept as empty compatibility gauges. ## Budget both intervals diff --git a/docs/store-isolation.md b/docs/store-isolation.md index c408df0..1ca0e90 100644 --- a/docs/store-isolation.md +++ b/docs/store-isolation.md @@ -43,8 +43,7 @@ Gateway uses `mode: gateway`, `grpc`, `health`, `prometheus`, `request`, and `fo | `forwarding.max_fanout` | `8` | Maximum parallel Store calls per public batch | These are process limits, not RSS guarantees or recommended resource requests. -Gateway charges actual input/result allocations to `memory`; growing responses -share a bounded completion reserve. The gRPC send ceiling determines response +Gateway checks process memory watermarks before forwarding new work. The gRPC send ceiling determines response capacity. Clients control request deadlines; shutdown has its own bounded drain. Routes belong to the same Gateway configuration: @@ -93,16 +92,14 @@ request-wide declarations, operation counts and completion modes before dispatch Same-record operations remain together and retain their order. There is no cross-Store transaction or global write ordering. -Reads and synchronous writes involving merge, conditional puts or returned -documents process Store groups in first-occurrence order. Snapshot, merge-input, -merge-output and returned-document budgets are independent. Each Engine receives -remaining grants and returns charges; coalesced requests retain separate grants. -CAS retries report their maximum charge per category. This conservative accounting -can reject a boundary-size cross-Store request that the old executor accepted. +Reads and synchronous writes requesting returned documents process Store groups in first-occurrence order. Only returned-document +allowances cross the forwarding boundary. Each Engine receives the remaining +response grant; coalesced requests retain their original RPC grants. Intermediate +snapshots and merge outputs have no separate byte quota. Returned-document space is checked **before committing**, never truncated after successful writes. Native requests apply the smaller of Gateway and Engine limits. -Unconditional puts without returned documents, deletes and async acceptance can +Writes without returned documents, deletes and async acceptance can forward Store groups concurrently within `max_fanout`. The extra network hop and serial budget-sensitive groups have a latency cost; benchmark your workload. @@ -137,10 +134,9 @@ not proxy named dependency health services; its default gRPC health is process h Gateway exports bounded method/code labels with `sink_gateway_requests_total`, `sink_gateway_request_duration_seconds`, `sink_gateway_engine_duration_seconds`, -`sink_gateway_in_flight_requests`, `sink_gateway_in_flight_bytes`, -`sink_gateway_rejected_total`, `sink_gateway_routes`, +`sink_gateway_in_flight_requests`, `sink_gateway_routes`, and `sink_gateway_config_info`. -Engine retains the existing `sink_grpc_server_*`, admission, batching and Lua +Engine retains the existing `sink_grpc_server_*`, in-flight request, batching and Lua metrics, labeled by the original public method even over private forwarding. Worker retains the existing pending/oldest/last-poll/last-commit/retry/DLQ metrics. Broker consumer lag comes from the external Kafka scaler or exporter. @@ -151,8 +147,8 @@ CPU, resource requests, replica floors and scale-to-zero are deployment settings Engine and Worker replica maxima and pool sizes must jointly fit their Store's backend capacity. Increasing Store A replicas creates no Store B database clients. Gateway remains a shared ingress: exhausting its CPU, memory or global admission -capacity can affect multiple Stores. Per-Store forwarding limits contain individual -backend pressure; Gateway still needs its own capacity planning and scaling. +pressure can affect multiple Stores. Gateway needs its own capacity planning and scaling. +All roles expose process [memory watermark metrics](observability.md#memory-capacity-and-keda). ## New-cluster deployment diff --git a/examples/autoscaling/memory-keda.yaml b/examples/autoscaling/memory-keda.yaml index a16c5dd..a82b9ab 100644 --- a/examples/autoscaling/memory-keda.yaml +++ b/examples/autoscaling/memory-keda.yaml @@ -26,12 +26,11 @@ spec: ignoreNullValues: "false" query: >- ( - (sum(sink_memory_used_bytes{namespace="sink",job="sink-gateway",role="gateway"}) - + sum(sink_memory_waiting_bytes{namespace="sink",job="sink-gateway",role="gateway"})) - / sum(sink_memory_capacity_bytes{namespace="sink",job="sink-gateway",role="gateway",pool="normal"}) + sum(sink_memory_used_bytes{namespace="sink",job="sink-gateway",role="gateway"}) + / sum(sink_memory_watermark_bytes{namespace="sink",job="sink-gateway",role="gateway",watermark="high"}) ) and on() (min(up{namespace="sink",job="sink-gateway"}) == 1) - and on() (count(sink_memory_capacity_bytes{namespace="sink",job="sink-gateway",role="gateway",pool="normal"}) + and on() (count(sink_memory_watermark_bytes{namespace="sink",job="sink-gateway",role="gateway",watermark="high"}) == count(up{namespace="sink",job="sink-gateway"})) - type: prometheus metricType: Value @@ -41,9 +40,9 @@ spec: ignoreNullValues: "false" query: >- ( - sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway",phase="request",reason="busy"}[1m])) + sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway"}[1m])) / clamp_min( - sum(rate(sink_memory_admitted_total{namespace="sink",job="sink-gateway",role="gateway",phase="request"}[1m])) - + sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway",phase="request",reason="busy"}[1m])), 1) + sum(rate(sink_memory_admitted_total{namespace="sink",job="sink-gateway",role="gateway"}[1m])) + + sum(rate(sink_memory_rejected_total{namespace="sink",job="sink-gateway",role="gateway"}[1m])), 1) ) and on() (min(up{namespace="sink",job="sink-gateway"}) == 1) diff --git a/examples/kubernetes/sink.yaml b/examples/kubernetes/sink.yaml index f7d0433..263a683 100644 --- a/examples/kubernetes/sink.yaml +++ b/examples/kubernetes/sink.yaml @@ -4,8 +4,6 @@ grpc: {address: ":8080", max_receive_message_bytes: 64MiB, max_send_message_byte health: {address: ":8081"} prometheus: {enabled: true, address: ":9090"} execution: - max_snapshot_bytes: 32MiB - max_output_bytes: 32MiB mongodb: {max_concurrent_writes: 64, max_concurrent_groups: 16} merge: {max_attempts: 3} batching: diff --git a/examples/quickstart/engine.yaml b/examples/quickstart/engine.yaml index dc87b9f..3c403eb 100644 --- a/examples/quickstart/engine.yaml +++ b/examples/quickstart/engine.yaml @@ -4,9 +4,9 @@ grpc: max_receive_message_bytes: 64MiB max_send_message_bytes: 64MiB memory: - # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. - burst_percent: 10 # Completion reserve, 1..99. - wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. + # max_bytes: 1GiB # Optional process ceiling; bounded by detected limits. + high_watermark_percent: 80 # Reject new work at this process memory usage. + low_watermark_percent: 70 # Resume new work after usage falls to this level. batching: max_wait: 2ms # Collection delay, not an execution timeout. max_operations: 32 # Measured small-document starting point; not the public RPC limit. @@ -17,11 +17,6 @@ batching: producer: max_buffered_bytes: 64MiB execution: - max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. - max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. - mongodb: # Only valid with the mongodb driver; tune independently in each role. - max_concurrent_writes: 64 - max_concurrent_groups: 16 merge: max_attempts: 3 # Revision-conflict attempts. lua: diff --git a/examples/quickstart/gateway.yaml b/examples/quickstart/gateway.yaml index 2cae75c..6fc4437 100644 --- a/examples/quickstart/gateway.yaml +++ b/examples/quickstart/gateway.yaml @@ -4,9 +4,9 @@ grpc: max_receive_message_bytes: 64MiB max_send_message_bytes: 64MiB memory: - # max_bytes: 256MiB # Omit to auto-detect managed capacity; not an RSS cap. - burst_percent: 10 # Completion reserve, 1..99. - wait_timeout: 2s # Memory-growth contention bound; never an overall RPC deadline. + # max_bytes: 1GiB # Optional process ceiling; bounded by detected limits. + high_watermark_percent: 80 # Reject new work at this process memory usage. + low_watermark_percent: 70 # Resume new work after usage falls to this level. request: max_operations: 1000 # Entire client request across all Stores. forwarding: diff --git a/examples/quickstart/store.yaml b/examples/quickstart/store.yaml index 814579c..fa51d7f 100644 --- a/examples/quickstart/store.yaml +++ b/examples/quickstart/store.yaml @@ -7,6 +7,8 @@ storage: uri: mongodb://mongo:27017/?replicaSet=rs0 # Example endpoint; required unless uri_file is set. # uri_file: /etc/sink-secrets/mongodb-uri # Omit uri when using its file alternative. metadata_field: __sink # Must match across roles and replicas. + max_concurrent_writes: 64 # Per process; shared by Engine and Worker configuration. + max_concurrent_groups: 16 # For elasticsearch/opensearch, replace mongodb with search: # search: # endpoints: [http://opensearch:9200] diff --git a/examples/quickstart/worker.yaml b/examples/quickstart/worker.yaml index cbadcf9..cfd00fb 100644 --- a/examples/quickstart/worker.yaml +++ b/examples/quickstart/worker.yaml @@ -1,7 +1,8 @@ mode: worker -# Only the managed capacity is configurable; reservation policy is internal. -# memory: -# max_bytes: 256MiB +memory: + # max_bytes: 1GiB # Optional process ceiling; bounded by detected limits. + high_watermark_percent: 80 + low_watermark_percent: 70 consumer: group_id: primary-workers max_poll_records: 500 @@ -11,11 +12,6 @@ consumer: backoff: 100ms max_backoff: 10s execution: - max_snapshot_bytes: 32MiB # Conditional-write snapshots, accumulated per execution owner. - max_output_bytes: 32MiB # Candidate documents, accumulated separately from snapshots. - mongodb: # Only valid with the mongodb driver; tune independently in each role. - max_concurrent_writes: 64 - max_concurrent_groups: 16 merge: max_attempts: 3 # Revision-conflict attempts. lua: diff --git a/gen/forward/forward.pb.go b/gen/forward/forward.pb.go index 2dac071..499b9b8 100644 --- a/gen/forward/forward.pb.go +++ b/gen/forward/forward.pb.go @@ -23,12 +23,10 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// Independent logical document budgets, including per-record envelope charges. +// Public response allowance, including per-record envelope charges. +// Intermediate allocations are owned by each process memory pool. type Budget struct { state protoimpl.MessageState `protogen:"open.v1"` - Snapshots uint64 `protobuf:"varint,1,opt,name=snapshots,proto3" json:"snapshots,omitempty"` - Inputs uint64 `protobuf:"varint,2,opt,name=inputs,proto3" json:"inputs,omitempty"` - Outputs uint64 `protobuf:"varint,3,opt,name=outputs,proto3" json:"outputs,omitempty"` Returns uint64 `protobuf:"varint,4,opt,name=returns,proto3" json:"returns,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -64,27 +62,6 @@ func (*Budget) Descriptor() ([]byte, []int) { return file_forward_forward_proto_rawDescGZIP(), []int{0} } -func (x *Budget) GetSnapshots() uint64 { - if x != nil { - return x.Snapshots - } - return 0 -} - -func (x *Budget) GetInputs() uint64 { - if x != nil { - return x.Inputs - } - return 0 -} - -func (x *Budget) GetOutputs() uint64 { - if x != nil { - return x.Outputs - } - return 0 -} - func (x *Budget) GetReturns() uint64 { if x != nil { return x.Returns @@ -554,12 +531,9 @@ var File_forward_forward_proto protoreflect.FileDescriptor const file_forward_forward_proto_rawDesc = "" + "\n" + - "\x15forward/forward.proto\x12\x0fsink.forward.v1\x1a\x0fsink/sink.proto\x1a\x19google/protobuf/any.proto\"r\n" + - "\x06Budget\x12\x1c\n" + - "\tsnapshots\x18\x01 \x01(\x04R\tsnapshots\x12\x16\n" + - "\x06inputs\x18\x02 \x01(\x04R\x06inputs\x12\x18\n" + - "\aoutputs\x18\x03 \x01(\x04R\aoutputs\x12\x18\n" + - "\areturns\x18\x04 \x01(\x04R\areturns\"\xc6\x03\n" + + "\x15forward/forward.proto\x12\x0fsink.forward.v1\x1a\x0fsink/sink.proto\x1a\x19google/protobuf/any.proto\"P\n" + + "\x06Budget\x12\x18\n" + + "\areturns\x18\x04 \x01(\x04R\areturnsJ\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04R\tsnapshotsR\x06inputsR\aoutputs\"\xc6\x03\n" + "\x0eForwardRequest\x12\x18\n" + "\aversion\x18\x01 \x01(\rR\aversion\x12\x14\n" + "\x05store\x18\x02 \x01(\tR\x05store\x12-\n" + diff --git a/gen/forward/forward_vtproto.pb.go b/gen/forward/forward_vtproto.pb.go index 93fccbf..d5792de 100644 --- a/gen/forward/forward_vtproto.pb.go +++ b/gen/forward/forward_vtproto.pb.go @@ -56,21 +56,6 @@ func (m *Budget) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x20 } - if m.Outputs != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Outputs)) - i-- - dAtA[i] = 0x18 - } - if m.Inputs != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Inputs)) - i-- - dAtA[i] = 0x10 - } - if m.Snapshots != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Snapshots)) - i-- - dAtA[i] = 0x8 - } return len(dAtA) - i, nil } @@ -557,15 +542,6 @@ func (m *Budget) SizeVT() (n int) { } var l int _ = l - if m.Snapshots != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Snapshots)) - } - if m.Inputs != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Inputs)) - } - if m.Outputs != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Outputs)) - } if m.Returns != 0 { n += 1 + protohelpers.SizeOfVarint(uint64(m.Returns)) } @@ -851,63 +827,6 @@ func (m *Budget) UnmarshalVT(dAtA []byte) error { return fmt.Errorf("proto: Budget: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Snapshots", wireType) - } - m.Snapshots = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Snapshots |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Inputs", wireType) - } - m.Inputs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Inputs |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Outputs", wireType) - } - m.Outputs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Outputs |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Returns", wireType) diff --git a/internal/app/app.go b/internal/app/app.go index 1a73aba..16a3f92 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -22,7 +22,7 @@ import ( ) type Application struct { - memory *capacity.Pool + memory *capacity.Guard draining atomic.Bool gateway *gateway.Server topics *queuekafka.TopicManager @@ -114,12 +114,3 @@ func New(ctx context.Context, opts Options) (*Application, error) { ready = true return app, nil } - -func newMemory(loaded config.Config) (*capacity.Pool, error) { - bytes, source := int64(loaded.Memory.MaxBytes), "configured" - if bytes == 0 { - bytes, source = capacity.Detect() - } - opts := capacity.Options{Bytes: bytes, BurstPercent: loaded.Memory.BurstPercent, WaitTimeout: loaded.Memory.WaitTimeout, Role: string(loaded.Mode), Store: loaded.Storage.Name, Source: source} - return capacity.New(opts) -} diff --git a/internal/app/kafka.go b/internal/app/kafka.go index 610cc0c..8024d33 100644 --- a/internal/app/kafka.go +++ b/internal/app/kafka.go @@ -55,7 +55,7 @@ func (app *Application) configureWorker(server *service.Server, observed *sinkme if err != nil { return err } - workerOptions := queuekafka.WorkerOptions{ + workerOptions := queuekafka.WorkerOptions{Memory: app.memory, ShutdownTimeout: loaded.ShutdownTimeout, Topics: app.topics, ProcessingTimeout: configured.Kafka.Consumer.ProcessingTimeout, MaxRecordBytes: configured.Kafka.MaxRecordBytes, diff --git a/internal/app/logging.go b/internal/app/logging.go index 927a7c2..e442d3f 100644 --- a/internal/app/logging.go +++ b/internal/app/logging.go @@ -8,7 +8,6 @@ import ( forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/protocol" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -21,11 +20,6 @@ func logUnary(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grp if method != "" { code := status.Code(err) observed := response - // Inspect the underlying message while returning the original wrapper; - // the codec still owns its reservation through response serialization. - if managed, ok := observed.(*protocol.ManagedMessage); ok { - observed = managed.Message - } if forwarded, ok := observed.(*forward.ForwardResponse); ok { if code == codes.OK { code = codes.Code(forwarded.GetCode()) diff --git a/internal/app/logging_test.go b/internal/app/logging_test.go index d1bc0d1..fdebde6 100644 --- a/internal/app/logging_test.go +++ b/internal/app/logging_test.go @@ -12,7 +12,6 @@ import ( sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/config" "github.com/liran/sink/internal/logging" - "github.com/liran/sink/internal/protocol" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -40,9 +39,6 @@ func TestRPCDiagnosticsCaptureApplicationFailuresWithoutPayload(t *testing.T) { wrapped := &forward.ForwardResponse_Write{Write: write} envelope := &forward.ForwardResponse{Response: wrapped} native := &sink.ExecuteResponse{Success: false, Payload: []byte("private response")} - managedWrite := &protocol.ManagedMessage{Message: write} - managedEnvelope := &protocol.ManagedMessage{Message: envelope} - managedNative := &protocol.ManagedMessage{Message: native} tests := []struct { name string response any @@ -50,9 +46,9 @@ func TestRPCDiagnosticsCaptureApplicationFailuresWithoutPayload(t *testing.T) { want string }{ {name: "partial failure", response: write, want: "warn"}, - {name: "managed failure", response: managedWrite, want: "warn"}, - {name: "managed forwarded failure", response: managedEnvelope, want: "warn"}, - {name: "managed native failure", response: managedNative, want: "warn"}, + {name: "managed failure", response: write, want: "warn"}, + {name: "managed forwarded failure", response: envelope, want: "warn"}, + {name: "managed native failure", response: native, want: "warn"}, {name: "forwarded failure", response: envelope, want: "warn"}, {name: "native failure", response: native, want: "warn"}, {name: "transport failure", err: status.Error(codes.Internal, "private error"), want: "error"}, diff --git a/internal/app/memory.go b/internal/app/memory.go new file mode 100644 index 0000000..786cf31 --- /dev/null +++ b/internal/app/memory.go @@ -0,0 +1,83 @@ +package app + +import ( + "fmt" + "math" + "strings" + + "github.com/liran/sink/internal/capacity" + "github.com/liran/sink/internal/config" +) + +type memoryRequirement struct { + name string + bytes int64 +} + +// These are startup sizing estimates, not allocation reservations or an OOM +// guarantee. Keep enough room below the high watermark for one normal work unit. +func minimumMemory(loaded config.Config) []memoryRequirement { + requirements := []memoryRequirement{{name: "runtime and drivers", bytes: 64 << 20}} + if loaded.Mode != config.ModeWorker { + requestMemory := memoryRequirement{name: "request decoding", bytes: startupBytes(2, loaded.GRPC.MaxReceiveMessageBytes)} + requirements = append(requirements, requestMemory) + responseMemory := memoryRequirement{name: "response encoding", bytes: startupBytes(2, loaded.GRPC.MaxSendMessageBytes)} + requirements = append(requirements, responseMemory) + } + if loaded.Mode != config.ModeGateway { + mergeMemory := memoryRequirement{name: "merge documents", bytes: startupBytes(2, loaded.Service.Merge.Lua.MaxResultBytes)} + requirements = append(requirements, mergeMemory) + sourceCache := memoryRequirement{name: "Lua source cache", bytes: startupBytes(loaded.Service.Merge.Lua.MaxCachedPrograms, loaded.Service.Merge.Lua.MaxSourceBytes)} + requirements = append(requirements, sourceCache) + if loaded.Storage.Driver == config.DriverMongoDB { + mongoMemory := memoryRequirement{name: "MongoDB wire buffers", bytes: 48 << 20} + requirements = append(requirements, mongoMemory) + } + if loaded.Storage.Kafka.Enabled { + if loaded.Mode == config.ModeEngine { + producerMemory := memoryRequirement{name: "Kafka producer buffer", bytes: int64(loaded.Storage.Kafka.Producer.MaxBufferedBytes)} + requirements = append(requirements, producerMemory) + } else { + fetchMemory := memoryRequirement{name: "Kafka fetch and decoded batch", bytes: 48 << 20} + requirements = append(requirements, fetchMemory) + deadLetterMemory := memoryRequirement{name: "Kafka dead-letter producer buffer", bytes: max(64<<20, int64(loaded.Storage.Kafka.MaxRecordBytes)+(16<<10))} + requirements = append(requirements, deadLetterMemory) + } + } + } + return requirements +} + +func newMemory(loaded config.Config) (*capacity.Guard, error) { + bytes, source := capacity.Detect() + if loaded.Memory.MaxBytes > 0 && (source == "fallback" || int64(loaded.Memory.MaxBytes) < bytes) { + bytes, source = int64(loaded.Memory.MaxBytes), "configured" + } + requireStartupMemory(loaded, bytes, source) + opts := capacity.Options{Bytes: bytes, HighPercent: loaded.Memory.HighWatermarkPercent, LowPercent: loaded.Memory.LowWatermarkPercent, Role: string(loaded.Mode), Store: loaded.Storage.Name, Source: source} + return capacity.New(opts) +} + +func requireStartupMemory(loaded config.Config, available int64, source string) { + var minimum int64 + var breakdown []string + for _, requirement := range minimumMemory(loaded) { + minimum += min(requirement.bytes, math.MaxInt64-minimum) + breakdown = append(breakdown, fmt.Sprintf("%s=%d bytes", requirement.name, requirement.bytes)) + } + high := int64(loaded.Memory.HighWatermarkPercent) + normal := available/100*high + available%100*high/100 + if normal < minimum { + panic(fmt.Sprintf("insufficient startup memory for %s: available=%d bytes (%s), high watermark=%d bytes (%d%%), minimum working memory=%d bytes; %s", loaded.Mode, available, source, normal, high, minimum, strings.Join(breakdown, ", "))) + } +} + +func startupBytes(count, size int) int64 { + if count <= 0 || size <= 0 { + return 0 + } + if int64(count) > math.MaxInt64/int64(size) { + return math.MaxInt64 + } + return int64(count) * int64(size) +} diff --git a/internal/app/memory_test.go b/internal/app/memory_test.go new file mode 100644 index 0000000..caf52fe --- /dev/null +++ b/internal/app/memory_test.go @@ -0,0 +1,67 @@ +package app + +import ( + "fmt" + "math" + "strings" + "testing" + + "github.com/liran/sink/internal/config" +) + +func TestStartupMemoryMinimumForEveryRole(t *testing.T) { + for _, role := range []string{"gateway", "engine", "worker"} { + t.Run(role, func(t *testing.T) { + component := "mode: " + role + "\n" + shared := strings.NewReader("name: primary\nstorage: {driver: mongodb, mongodb: {uri: 'mongodb://localhost:27017'}}\nkafka: {enabled: true, brokers: [localhost:9092], topic: {name: mutations}}") + if role == "gateway" { + component += "forwarding: {routes: [{store: primary, target: 'localhost:8080', tls: {insecure: true}}]}\n" + shared = nil + } + if role == "worker" { + component += "consumer: {group_id: workers}\n" + } + // A typed nil io.Reader is not an absent Store document. + var loaded config.Config + var err error + if shared == nil { + loaded, err = config.Decode(strings.NewReader(component), nil) + } else { + loaded, err = config.Decode(strings.NewReader(component), shared) + } + if err != nil { + t.Fatal(err) + } + var minimum int64 + for _, item := range minimumMemory(loaded) { + minimum += item.bytes + } + enough := (minimum*100 + 79) / 80 + requireStartupMemory(loaded, enough, "test") + defer func() { + message := fmt.Sprint(recover()) + for _, part := range []string{"insufficient startup memory", role, "available=", "minimum working memory=", "runtime and drivers="} { + if !strings.Contains(message, part) { + t.Fatalf("missing startup diagnostic %q: %s", part, message) + } + } + }() + requireStartupMemory(loaded, enough-1, "test") + }) + } +} + +func TestStartupMemoryEstimateCannotOverflow(t *testing.T) { + loaded, err := config.Decode(strings.NewReader("mode: gateway\nforwarding: {routes: [{store: primary, target: 'localhost:8080', tls: {insecure: true}}]}"), nil) + if err != nil { + t.Fatal(err) + } + loaded.GRPC.MaxReceiveMessageBytes = math.MaxInt + loaded.GRPC.MaxSendMessageBytes = math.MaxInt + defer func() { + if recover() == nil { + t.Fatal("overflow bypassed the startup memory check") + } + }() + requireStartupMemory(loaded, math.MaxInt64, "test") +} diff --git a/internal/app/service.go b/internal/app/service.go index 1bd0d89..6c1a1aa 100644 --- a/internal/app/service.go +++ b/internal/app/service.go @@ -23,10 +23,7 @@ func (app *Application) newService(observed *sinkmetrics.Metrics) (*service.Serv return nil, err } serverOptions := service.Options{ - Memory: app.memory, - MaxReadBytes: max(loaded.GRPC.MaxSendMessageBytes, loaded.Service.Execution.MaxOutputBytes), - MaxSnapshotBytes: loaded.Service.Execution.MaxSnapshotBytes, - MaxOutputBytes: loaded.Service.Execution.MaxOutputBytes, + MaxReadBytes: loaded.GRPC.MaxSendMessageBytes, Storage: app.storage, Lua: luaEngine, Publisher: app.publisher, @@ -57,7 +54,7 @@ func (app *Application) configureServer(sinkServer *service.Server, observed *si if err := app.configureGRPC(app.batchingServer, observed); err != nil { return err } - opts := engine.Options{MaxRequestBytes: loaded.GRPC.MaxReceiveMessageBytes, Metrics: observed, Service: app.batchingServer, Store: loaded.Storage.Name, MaxReadBytes: loaded.GRPC.MaxSendMessageBytes} + opts := engine.Options{Memory: app.memory, MaxRequestBytes: loaded.GRPC.MaxReceiveMessageBytes, Metrics: observed, Service: app.batchingServer, Store: loaded.Storage.Name, MaxReadBytes: loaded.GRPC.MaxSendMessageBytes} forwardingServer, err := engine.New(opts) if err != nil { return err diff --git a/internal/app/transport.go b/internal/app/transport.go index cbad359..de8f943 100644 --- a/internal/app/transport.go +++ b/internal/app/transport.go @@ -26,10 +26,6 @@ func (app *Application) configureGRPC(server sink.SinkServer, observed *sinkmetr return fmt.Errorf("listen for gRPC: %w", err) } serverOptions := make([]grpc.ServerOption, 0, 4) - memoryStats := &protocol.MemoryStats{Pool: app.memory} - if app.memory != nil { - serverOptions = append(serverOptions, grpc.StatsHandler(memoryStats)) - } overhead := 0 if app.config.Mode == config.ModeEngine { overhead = forwarding.EnvelopeBytes @@ -41,9 +37,6 @@ func (app *Application) configureGRPC(server sink.SinkServer, observed *sinkmetr interceptors := make([]grpc.UnaryServerInterceptor, 0, 2) interceptors = append(interceptors, logUnary) streamInterceptors := []grpc.StreamServerInterceptor{logStream} - if app.memory != nil { - interceptors = append(interceptors, protocol.MemoryInterceptor) - } if app.gateway != nil { interceptors = append(interceptors, app.gateway.UnaryInterceptor()) } diff --git a/internal/capacity/detect.go b/internal/capacity/detect.go index 781b51e..b1bc941 100644 --- a/internal/capacity/detect.go +++ b/internal/capacity/detect.go @@ -10,10 +10,7 @@ import ( "strings" ) -const DefaultBurstPercent = 10 - -// Detect leaves half of the effective memory limit for runtime, driver-owned -// buffers, ingress, caches and GC. Managed bytes are deliberately not RSS. +// Detect finds the effective process ceiling from Go, cgroups, or host memory. func Detect() (int64, string) { return detect(os.DirFS("/"), debug.SetMemoryLimit(-1)) } @@ -100,9 +97,9 @@ func detect(files fs.FS, goLimit int64) (int64, string) { } } if limit == math.MaxInt64 { - return 256 << 20, "fallback" + return 1 << 30, "fallback" } - return max(1024, limit/2), source + return max(1024, limit), source } func containsController(value, controller string) bool { diff --git a/internal/capacity/detect_test.go b/internal/capacity/detect_test.go new file mode 100644 index 0000000..fee41c8 --- /dev/null +++ b/internal/capacity/detect_test.go @@ -0,0 +1,32 @@ +package capacity + +import ( + "math" + "testing" + "testing/fstest" +) + +func TestDetectCapacity(t *testing.T) { + for _, tc := range []struct { + name, membership, mount, filename, value string + soft, want int64 + source string + }{ + {"v2-parent", "0::/pod/child", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/pod/memory.max", "104857600", math.MaxInt64, 100 << 20, "cgroup"}, + {"v1", "1:cpu,memory:/pod", "1 0 0:1 / /sys/fs/cgroup/memory rw - cgroup cgroup rw,memory", "sys/fs/cgroup/memory/pod/memory.limit_in_bytes", "209715200", math.MaxInt64, 200 << 20, "cgroup"}, + {"namespaced", "0::/", "1 0 0:1 /pod /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/memory.max", "104857600", math.MaxInt64, 100 << 20, "cgroup"}, + {"soft", "0::/", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/memory.max", "104857600", 40 << 20, 40 << 20, "gomemlimit"}, + {"unlimited", "0::/", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/memory.max", "max", math.MaxInt64, 1 << 30, "fallback"}, + } { + t.Run(tc.name, func(t *testing.T) { + files := fstest.MapFS{} + files["proc/self/cgroup"] = &fstest.MapFile{Data: []byte(tc.membership)} + files["proc/self/mountinfo"] = &fstest.MapFile{Data: []byte(tc.mount)} + files[tc.filename] = &fstest.MapFile{Data: []byte(tc.value)} + got, source := detect(files, tc.soft) + if got != tc.want || source != tc.source { + t.Fatalf("got %d %s, want %d %s", got, source, tc.want, tc.source) + } + }) + } +} diff --git a/internal/capacity/guard.go b/internal/capacity/guard.go new file mode 100644 index 0000000..05cc264 --- /dev/null +++ b/internal/capacity/guard.go @@ -0,0 +1,143 @@ +// Package capacity gates new work using process memory high and low watermarks. +package capacity + +import ( + "context" + "errors" + "io/fs" + "math" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const SampleInterval = 100 * time.Millisecond +const DefaultHighPercent = 80 +const DefaultLowPercent = 70 + +var ErrBusy = status.Error(codes.ResourceExhausted, "process memory is above its admission watermark") + +type Options struct { + Bytes int64 + HighPercent int + LowPercent int + Role string + Store string + Source string + // Files supplies the operating-system memory counters; nil uses the host filesystem. + Files fs.FS +} + +type Guard struct { + mu sync.Mutex + limit int64 + high int64 + low int64 + used int64 + blocked bool + sampled time.Time + files fs.FS + usageSource string + limitSource string + observed *observations +} + +type contextKey struct{} + +func New(opts Options) (*Guard, error) { + if opts.Bytes < 1024 || opts.HighPercent < 1 || opts.HighPercent > 99 || opts.LowPercent < 1 || opts.LowPercent >= opts.HighPercent { + return nil, errors.New("memory requires at least 1KiB and 0 < low_watermark_percent < high_watermark_percent < 100") + } + files := opts.Files + if files == nil { + files = os.DirFS("/") + } + guard := &Guard{limit: opts.Bytes, high: percent(opts.Bytes, opts.HighPercent), low: percent(opts.Bytes, opts.LowPercent), files: files, limitSource: opts.Source} + guard.observed = newObservations(opts) + return guard, nil +} + +func percent(bytes int64, value int) int64 { + return bytes/100*int64(value) + bytes%100*int64(value)/100 +} + +// Admit checks once at the process boundary. Already admitted work is never +// rechecked while queued, executing, returning a response, or retrying a conflict. +func (g *Guard) Admit(ctx context.Context) (context.Context, error) { + if err := ctx.Err(); err != nil { + return ctx, status.FromContextError(err).Err() + } + if g == nil || ctx.Value(contextKey{}) == g { + return ctx, nil + } + _, blocked, _ := g.snapshot() + if blocked { + g.observed.rejected.Inc() + return ctx, ErrBusy + } + g.observed.admitted.Inc() + return context.WithValue(ctx, contextKey{}, g), nil +} + +func (g *Guard) Blocked() bool { + if g == nil { + return false + } + _, blocked, _ := g.snapshot() + return blocked +} + +// Wait is used before a Worker polls its next batch; it never interrupts a batch. +func (g *Guard) Wait(ctx context.Context) error { + if g == nil { + return ctx.Err() + } + ticker := time.NewTicker(SampleInterval) + defer ticker.Stop() + for g.Blocked() { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + return ctx.Err() +} + +func (g *Guard) snapshot() (int64, bool, string) { + g.mu.Lock() + defer g.mu.Unlock() + if g.sampled.IsZero() || time.Since(g.sampled) >= SampleInterval { + g.used, g.usageSource = processUsage(g.files) + g.sampled = time.Now() + if g.used >= g.high { + g.blocked = true + } else if g.used <= g.low { + g.blocked = false + } + } + return g.used, g.blocked, g.usageSource +} + +func processUsage(files fs.FS) (int64, string) { + if raw, err := fs.ReadFile(files, "proc/self/statm"); err == nil { + fields := strings.Fields(string(raw)) + if len(fields) >= 2 { + pages, err := strconv.ParseInt(fields[1], 10, 64) + if err == nil && pages >= 0 && pages <= math.MaxInt64/int64(os.Getpagesize()) { + return pages * int64(os.Getpagesize()), "rss" + } + } + } + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + return int64(min(stats.Sys-stats.HeapReleased, math.MaxInt64)), "go_runtime" +} + +func (g *Guard) Limit() int64 { return g.limit } diff --git a/internal/capacity/guard_test.go b/internal/capacity/guard_test.go new file mode 100644 index 0000000..c5ddfab --- /dev/null +++ b/internal/capacity/guard_test.go @@ -0,0 +1,142 @@ +package capacity + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "testing" + "testing/fstest" + "testing/synctest" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestWatermarksOnlyGateNewWork(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + files := fstest.MapFS{} + opts := Options{Bytes: int64(100 * os.Getpagesize()), HighPercent: 80, LowPercent: 70, Files: files} + guard, err := New(opts) + if err != nil { + t.Fatal(err) + } + set := func(pages int) { + files["proc/self/statm"] = &fstest.MapFile{Data: []byte(fmt.Sprintf("100 %d", pages))} + time.Sleep(SampleInterval) + } + set(79) + admitted, err := guard.Admit(t.Context()) + if err != nil { + t.Fatal(err) + } + set(80) + if _, err := guard.Admit(t.Context()); !errors.Is(err, ErrBusy) { + t.Fatalf("high watermark admitted new work: %v", err) + } + if _, err := guard.Admit(admitted); err != nil { + t.Fatalf("admitted work was interrupted: %v", err) + } + set(75) + if !guard.Blocked() { + t.Fatal("recovered before the low watermark") + } + set(70) + if _, err := guard.Admit(t.Context()); err != nil { + t.Fatalf("low watermark did not recover: %v", err) + } + if admitted.Err() != nil { + t.Fatal("pressure canceled an existing request") + } + canceled, cancel := context.WithCancel(admitted) + cancel() + if _, err := guard.Admit(canceled); err == nil { + t.Fatal("caller cancellation ignored") + } + }) +} + +func TestWatermarkWaitRecoveryAndCancellation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + files := fstest.MapFS{"proc/self/statm": {Data: []byte("100 90")}} + opts := Options{Bytes: int64(100 * os.Getpagesize()), HighPercent: 80, LowPercent: 70, Files: files} + guard, err := New(opts) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + ctx, cancel := context.WithCancel(t.Context()) + go func() { done <- guard.Wait(ctx) }() + synctest.Wait() + select { + case <-done: + t.Fatal("pressure did not pause polling") + default: + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + go func() { done <- guard.Wait(t.Context()) }() + synctest.Wait() + guard.mu.Lock() + files["proc/self/statm"] = &fstest.MapFile{Data: []byte("100 69")} + guard.mu.Unlock() + time.Sleep(SampleInterval) + if err := <-done; err != nil { + t.Fatal(err) + } + }) +} + +func TestConcurrentAdmissionAndMetrics(t *testing.T) { + files := fstest.MapFS{"proc/self/statm": {Data: []byte("100 90")}} + opts := Options{Bytes: int64(100 * os.Getpagesize()), HighPercent: 80, LowPercent: 70, Files: files, Role: "engine", Store: "primary", Source: "configured"} + guard, err := New(opts) + if err != nil { + t.Fatal(err) + } + var calls sync.WaitGroup + for range 32 { + calls.Go(func() { + if _, err := guard.Admit(t.Context()); !errors.Is(err, ErrBusy) { + t.Errorf("unexpected admission: %v", err) + } + }) + } + calls.Wait() + registry := prometheus.NewRegistry() + registry.MustRegister(guard) + metrics, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + found := map[string]bool{} + for _, family := range metrics { + found[family.GetName()] = true + } + if len(found) != 7 { + t.Fatalf("obsolete or unexpected memory metrics: %v", found) + } + for _, name := range []string{"sink_memory_limit_bytes", "sink_memory_used_bytes", "sink_memory_watermark_bytes", "sink_memory_pressure", "sink_memory_admitted_total", "sink_memory_rejected_total", "sink_memory_limit_source_info"} { + if !found[name] { + t.Fatalf("missing metric %s", name) + } + } +} + +func TestMemoryCounterFallbackAndValidation(t *testing.T) { + for _, raw := range []string{"", "100 invalid", "100 -1", "100 9223372036854775807"} { + files := fstest.MapFS{"proc/self/statm": {Data: []byte(raw)}} + used, source := processUsage(files) + if source != "go_runtime" || used <= 0 { + t.Fatalf("invalid fallback: %d %s", used, source) + } + } + for _, opts := range []Options{{Bytes: 1024, HighPercent: 80, LowPercent: 80}, {Bytes: 1024, HighPercent: 100, LowPercent: 70}, {Bytes: 512, HighPercent: 80, LowPercent: 70}, {Bytes: 1024, HighPercent: 80, LowPercent: 0}} { + if _, err := New(opts); err == nil { + t.Fatal("invalid memory configuration accepted") + } + } +} diff --git a/internal/capacity/metrics.go b/internal/capacity/metrics.go index a64874a..c66c7c1 100644 --- a/internal/capacity/metrics.go +++ b/internal/capacity/metrics.go @@ -1,93 +1,50 @@ package capacity -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" -) +import "github.com/prometheus/client_golang/prometheus" type observations struct { - admitted *prometheus.CounterVec - capacity *prometheus.Desc - opaque *prometheus.Desc - used *prometheus.Desc - waitingBytes *prometheus.Desc - waitingRequests *prometheus.Desc - oldest *prometheus.Desc - borrowers *prometheus.Desc - source *prometheus.Desc - rejected *prometheus.CounterVec - wait *prometheus.HistogramVec + limit *prometheus.Desc + used *prometheus.Desc + watermark *prometheus.Desc + blocked *prometheus.Desc + source *prometheus.Desc + admitted prometheus.Counter + rejected prometheus.Counter } func newObservations(opts Options) *observations { labels := prometheus.Labels{"role": opts.Role, "store": opts.Store} - observed := &observations{} - observed.opaque = prometheus.NewDesc("sink_memory_opaque_reserved_bytes", "Temporary capacity reserved around driver-owned wire allocations; included in used bytes.", nil, labels) - observed.capacity = prometheus.NewDesc("sink_memory_capacity_bytes", "Managed capacity; burst is reserved for completing admitted work.", []string{"pool"}, labels) - observed.used = prometheus.NewDesc("sink_memory_used_bytes", "Owned payload, copy and opaque driver capacity, not process RSS.", []string{"pool"}, labels) - observed.waitingBytes = prometheus.NewDesc("sink_memory_waiting_bytes", "Additional bytes requested by capacity waiters.", []string{"phase"}, labels) - observed.waitingRequests = prometheus.NewDesc("sink_memory_waiting_requests", "Number of capacity acquisitions waiting, not a request concurrency limit.", []string{"phase"}, labels) - observed.oldest = prometheus.NewDesc("sink_memory_oldest_wait_seconds", "Age of the oldest capacity waiter.", []string{"phase"}, labels) - observed.borrowers = prometheus.NewDesc("sink_memory_burst_borrowers", "Operations owning the completion reserve, zero or one.", nil, labels) - sourceLabels := prometheus.Labels{"role": opts.Role, "store": opts.Store, "source": opts.Source} - observed.source = prometheus.NewDesc("sink_memory_capacity_source_info", "Source of the resolved process memory capacity.", nil, sourceLabels) - admittedOpts := prometheus.CounterOpts{Name: "sink_memory_admitted_total", Help: "Successful capacity acquisitions by phase.", ConstLabels: labels} - observed.admitted = prometheus.NewCounterVec(admittedOpts, []string{"phase"}) - rejectedOpts := prometheus.CounterOpts{Name: "sink_memory_rejected_total", Help: "Capacity acquisition failures by phase and reason.", ConstLabels: labels} - observed.rejected = prometheus.NewCounterVec(rejectedOpts, []string{"phase", "reason"}) - waitOpts := prometheus.HistogramOpts{Name: "sink_memory_wait_seconds", Help: "Time waiting for process memory capacity.", ConstLabels: labels, Buckets: prometheus.DefBuckets} - observed.wait = prometheus.NewHistogramVec(waitOpts, []string{"phase", "outcome"}) - for _, phase := range []Phase{Request, Response} { - observed.admitted.WithLabelValues(string(phase)).Add(0) - for _, reason := range []string{"oversize", "busy", "wait_timeout"} { - observed.rejected.WithLabelValues(string(phase), reason).Add(0) - } + admittedOpts := prometheus.CounterOpts{Name: "sink_memory_admitted_total", Help: "New requests admitted below the memory watermarks.", ConstLabels: labels} + rejectedOpts := prometheus.CounterOpts{Name: "sink_memory_rejected_total", Help: "New requests rejected by memory pressure before execution.", ConstLabels: labels} + observed := &observations{ + limit: prometheus.NewDesc("sink_memory_limit_bytes", "Effective process memory ceiling used to calculate admission watermarks.", nil, labels), + used: prometheus.NewDesc("sink_memory_used_bytes", "Observed process memory; RSS on Linux, Go runtime memory when RSS is unavailable.", []string{"source"}, labels), + watermark: prometheus.NewDesc("sink_memory_watermark_bytes", "High rejection and low recovery watermarks.", []string{"watermark"}, labels), + blocked: prometheus.NewDesc("sink_memory_pressure", "Whether new work is paused by the high/low memory watermark policy.", nil, labels), + source: prometheus.NewDesc("sink_memory_limit_source_info", "Source of the effective memory ceiling.", []string{"source"}, labels), + admitted: prometheus.NewCounter(admittedOpts), rejected: prometheus.NewCounter(rejectedOpts), } return observed } - -func (p *Pool) Describe(ch chan<- *prometheus.Desc) { - o := p.observed - for _, desc := range []*prometheus.Desc{o.opaque, o.capacity, o.used, o.waitingBytes, o.waitingRequests, o.oldest, o.borrowers, o.source} { +func (g *Guard) Describe(ch chan<- *prometheus.Desc) { + for _, desc := range []*prometheus.Desc{g.observed.limit, g.observed.used, g.observed.watermark, g.observed.blocked, g.observed.source} { ch <- desc } - o.admitted.Describe(ch) - o.rejected.Describe(ch) - o.wait.Describe(ch) + g.observed.admitted.Describe(ch) + g.observed.rejected.Describe(ch) } - -func (p *Pool) Collect(ch chan<- prometheus.Metric) { - p.mu.Lock() - opaque := p.opaque - used, normal, total := p.used, p.normal, p.total - borrowers := 0.0 - if p.burstOwner != nil { - borrowers = 1 - } - requests := map[Phase]int64{Request: 0, Response: 0} - bytes := map[Phase]int64{Request: 0, Response: 0} - oldest := map[Phase]float64{Request: 0, Response: 0} - for _, w := range p.waiters { - requests[w.phase]++ - bytes[w.phase] += w.bytes - oldest[w.phase] = max(oldest[w.phase], time.Since(w.started).Seconds()) - } - p.mu.Unlock() - o := p.observed - ch <- prometheus.MustNewConstMetric(o.opaque, prometheus.GaugeValue, float64(opaque)) - ch <- prometheus.MustNewConstMetric(o.capacity, prometheus.GaugeValue, float64(normal), "normal") - ch <- prometheus.MustNewConstMetric(o.capacity, prometheus.GaugeValue, float64(total-normal), "burst") - ch <- prometheus.MustNewConstMetric(o.used, prometheus.GaugeValue, float64(min(used, normal)), "normal") - ch <- prometheus.MustNewConstMetric(o.used, prometheus.GaugeValue, float64(max(0, used-normal)), "burst") - ch <- prometheus.MustNewConstMetric(o.borrowers, prometheus.GaugeValue, borrowers) - ch <- prometheus.MustNewConstMetric(o.source, prometheus.GaugeValue, 1) - for _, phase := range []Phase{Request, Response} { - ch <- prometheus.MustNewConstMetric(o.waitingRequests, prometheus.GaugeValue, float64(requests[phase]), string(phase)) - ch <- prometheus.MustNewConstMetric(o.waitingBytes, prometheus.GaugeValue, float64(bytes[phase]), string(phase)) - ch <- prometheus.MustNewConstMetric(o.oldest, prometheus.GaugeValue, oldest[phase], string(phase)) +func (g *Guard) Collect(ch chan<- prometheus.Metric) { + used, blocked, source := g.snapshot() + pressure := 0.0 + if blocked { + pressure = 1 } - o.admitted.Collect(ch) - o.rejected.Collect(ch) - o.wait.Collect(ch) + ch <- prometheus.MustNewConstMetric(g.observed.limit, prometheus.GaugeValue, float64(g.limit)) + ch <- prometheus.MustNewConstMetric(g.observed.used, prometheus.GaugeValue, float64(used), source) + ch <- prometheus.MustNewConstMetric(g.observed.watermark, prometheus.GaugeValue, float64(g.high), "high") + ch <- prometheus.MustNewConstMetric(g.observed.watermark, prometheus.GaugeValue, float64(g.low), "low") + ch <- prometheus.MustNewConstMetric(g.observed.blocked, prometheus.GaugeValue, pressure) + ch <- prometheus.MustNewConstMetric(g.observed.source, prometheus.GaugeValue, 1, g.limitSource) + g.observed.admitted.Collect(ch) + g.observed.rejected.Collect(ch) } diff --git a/internal/capacity/pool.go b/internal/capacity/pool.go deleted file mode 100644 index ebdf129..0000000 --- a/internal/capacity/pool.go +++ /dev/null @@ -1,297 +0,0 @@ -// Package capacity manages process-local ownership of request and response bytes. -package capacity - -import ( - "context" - "errors" - "sync" - "time" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -var ErrBusy = status.Error(codes.ResourceExhausted, "process memory capacity is occupied") - -type Phase string - -const ( - Request Phase = "request" - Response Phase = "response" -) - -type Options struct { - Bytes int64 - BurstPercent int - WaitTimeout time.Duration - MaxWaiters int - Source string - Role string - Store string -} - -type Pool struct { - mu sync.Mutex - total int64 - normal int64 - used int64 - opaque int64 - burstOwner *Owner - waiters []*waiter - waitTimeout time.Duration - maxWaiters int - observed *observations -} - -// Owner identifies one operation, including all of its retained buffers. -type Owner struct { - pool *Pool - bytes int64 -} - -// Lease is independently released when the corresponding buffer stops being owned. -type Lease struct { - owner *Owner - growMu sync.Mutex - bytes int64 - closed bool - opaque bool -} - -type waiter struct { - lease *Lease - bytes int64 - phase Phase - started time.Time - ready chan struct{} - granted bool -} - -func New(opts Options) (*Pool, error) { - if opts.Bytes < 1024 || opts.BurstPercent < 1 || opts.BurstPercent > 99 || opts.WaitTimeout <= 0 { - return nil, errors.New("memory capacity requires at least 1KiB, burst_percent between 1 and 99, and a positive wait timeout") - } - burst := opts.Bytes / 100 * int64(opts.BurstPercent) - if opts.MaxWaiters == 0 { - opts.MaxWaiters = 1024 - } - if opts.MaxWaiters < 0 { - return nil, errors.New("memory max_waiters must be positive") - } - pool := &Pool{total: opts.Bytes, normal: opts.Bytes - burst, waitTimeout: opts.WaitTimeout, maxWaiters: opts.MaxWaiters} - pool.observed = newObservations(opts) - return pool, nil -} - -func (p *Pool) NewOwner() *Owner { - owner := &Owner{pool: p} - return owner -} - -func (o *Owner) NewLease() *Lease { - lease := &Lease{owner: o} - return lease -} - -func (p *Pool) Limit() int64 { return p.total } - -func (l *Lease) Bytes() int64 { - p := l.owner.pool - p.mu.Lock() - defer p.mu.Unlock() - return l.bytes -} - -// Grow acquires the entire allocation before allowing its caller to allocate. -// Response growth has priority over new requests. Only one owner may borrow -// the completion reserve, so it is not fragmented among unrelated operations. -func (l *Lease) Grow(ctx context.Context, bytes int64, phase Phase) error { - l.growMu.Lock() - defer l.growMu.Unlock() - if bytes < 0 || (phase != Request && phase != Response) { - return errors.New("invalid memory capacity acquisition") - } - if err := ctx.Err(); err != nil { - return status.FromContextError(err).Err() - } - p := l.owner.pool - p.mu.Lock() - if l.closed { - p.mu.Unlock() - return errors.New("memory lease is closed") - } - limit := p.total - if phase == Request { - limit = p.normal - } - if bytes > limit-l.owner.bytes { - p.observed.rejected.WithLabelValues(string(phase), "oversize").Inc() - p.mu.Unlock() - return status.Error(codes.ResourceExhausted, "allocation exceeds process memory capacity") - } - if bytes == 0 { - p.mu.Unlock() - return nil - } - w := &waiter{lease: l, bytes: bytes, phase: phase, started: time.Now(), ready: make(chan struct{})} - p.waiters = append(p.waiters, w) - p.dispatch() - if w.granted { - p.observed.admitted.WithLabelValues(string(phase)).Inc() - p.mu.Unlock() - return nil - } - // Decoded arrivals must not accumulate outside the managed capacity. The - // batch queue waits only after acquiring ordinary bytes. Responses may wait - // because they already own an admitted request and need to finish it. - if phase == Request || len(p.waiters) > p.maxWaiters { - p.remove(w) - p.observed.rejected.WithLabelValues(string(phase), "busy").Inc() - p.mu.Unlock() - return ErrBusy - } - p.mu.Unlock() - waitCtx, cancel := context.WithTimeout(ctx, p.waitTimeout) - defer cancel() - select { - case <-w.ready: - case <-waitCtx.Done(): - } - p.mu.Lock() - defer p.mu.Unlock() - if l.closed { - return status.Error(codes.Canceled, "memory lease closed while waiting") - } - if err := waitCtx.Err(); err != nil { - if w.granted { - l.releaseLocked(min(bytes, l.bytes)) - } else { - p.remove(w) - } - p.dispatch() - p.observed.wait.WithLabelValues(string(phase), "canceled").Observe(time.Since(w.started).Seconds()) - if ctx.Err() != nil { - return status.FromContextError(ctx.Err()).Err() - } - p.observed.rejected.WithLabelValues(string(phase), "wait_timeout").Inc() - return status.Error(codes.ResourceExhausted, "timed out waiting for process memory capacity") - } - if !w.granted { - p.remove(w) - p.dispatch() - return status.Error(codes.Canceled, "memory lease closed while waiting") - } - p.observed.admitted.WithLabelValues(string(phase)).Inc() - p.observed.wait.WithLabelValues(string(phase), "admitted").Observe(time.Since(w.started).Seconds()) - return nil -} - -func (p *Pool) remove(target *waiter) { - for index, w := range p.waiters { - if w == target { - copy(p.waiters[index:], p.waiters[index+1:]) - p.waiters[len(p.waiters)-1] = nil - p.waiters = p.waiters[:len(p.waiters)-1] - return - } - } -} - -func (p *Pool) dispatch() { - for _, phase := range []Phase{Response, Request} { - pendingResponse := false - for index := 0; index < len(p.waiters); { - w := p.waiters[index] - if w.phase != phase { - index++ - continue - } - limit := p.normal - if phase == Response && (p.burstOwner == nil || p.burstOwner == w.lease.owner) { - limit = p.total - } - if w.bytes > limit-p.used { - pendingResponse = phase == Response - index++ - continue - } - if p.used+w.bytes > p.normal { - p.burstOwner = w.lease.owner - } - p.used += w.bytes - if w.lease.opaque { - p.opaque += w.bytes - } - w.lease.bytes += w.bytes - w.lease.owner.bytes += w.bytes - w.granted = true - p.remove(w) - close(w.ready) - } - if pendingResponse { - return - } - } -} - -func (l *Lease) releaseLocked(bytes int64) { - p := l.owner.pool - l.bytes -= bytes - l.owner.bytes -= bytes - p.used -= bytes - if l.opaque { - p.opaque -= bytes - } - if p.used <= p.normal || (p.burstOwner == l.owner && l.owner.bytes == 0) { - p.burstOwner = nil - } -} - -func (l *Lease) Shrink(bytes int64) { - p := l.owner.pool - p.mu.Lock() - defer p.mu.Unlock() - if l.closed || bytes <= 0 { - return - } - l.releaseLocked(min(bytes, l.bytes)) - p.dispatch() -} - -func (l *Lease) Close() { - p := l.owner.pool - p.mu.Lock() - defer p.mu.Unlock() - if l.closed { - return - } - l.closed = true - l.releaseLocked(l.bytes) - for index := 0; index < len(p.waiters); { - w := p.waiters[index] - if w.lease != l { - index++ - continue - } - p.remove(w) - close(w.ready) - } - p.dispatch() -} - -func (p *Pool) Used() int64 { p.mu.Lock(); defer p.mu.Unlock(); return p.used } - -// MarkOpaque distinguishes capacity protecting an uninstrumented driver from -// document and transport allocations. Call it before growing a new lease. -func (l *Lease) MarkOpaque() { - if l == nil { - return - } - p := l.owner.pool - p.mu.Lock() - defer p.mu.Unlock() - if !l.opaque { - l.opaque = true - p.opaque += l.bytes - } -} diff --git a/internal/capacity/pool_test.go b/internal/capacity/pool_test.go deleted file mode 100644 index f06297d..0000000 --- a/internal/capacity/pool_test.go +++ /dev/null @@ -1,296 +0,0 @@ -package capacity - -import ( - "context" - "math" - "sync" - "testing" - "testing/fstest" - "time" - - "github.com/prometheus/client_golang/prometheus" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func testPool(t testing.TB, percent int) *Pool { - t.Helper() - opts := Options{Bytes: 10000, BurstPercent: percent, WaitTimeout: time.Second} - p, err := New(opts) - if err != nil { - t.Fatal(err) - } - return p -} -func grow(t testing.TB, l *Lease, bytes int64, phase Phase) { - t.Helper() - if err := l.Grow(context.Background(), bytes, phase); err != nil { - t.Fatal(err) - } -} -func awaitWaiters(t *testing.T, p *Pool, count int) { - t.Helper() - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - p.mu.Lock() - n := len(p.waiters) - p.mu.Unlock() - if n == count { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("did not get %d waiters", count) -} -func assertEmpty(t testing.TB, p *Pool) { - t.Helper() - p.mu.Lock() - defer p.mu.Unlock() - if p.used != 0 || len(p.waiters) != 0 || p.burstOwner != nil { - t.Fatalf("leaked capacity: used=%d waiting=%d borrower=%v", p.used, len(p.waiters), p.burstOwner != nil) - } -} - -func TestResponsePriorityAndExclusiveReserve(t *testing.T) { - p := testPool(t, 10) - first, second := p.NewOwner().NewLease(), p.NewOwner().NewLease() - grow(t, first, 5000, Request) - grow(t, second, 4000, Request) - newcomer := p.NewOwner().NewLease() - if status.Code(newcomer.Grow(t.Context(), 1, Request)) != codes.ResourceExhausted { - t.Fatal("request borrowed completion reserve") - } - grow(t, first, 500, Response) - done := make(chan error, 1) - go func() { done <- second.Grow(t.Context(), 700, Response) }() - awaitWaiters(t, p, 1) - // Even when some ordinary capacity becomes free, a queued response wins. - first.Shrink(1000) - awaitWaiters(t, p, 0) - if err := <-done; err != nil { - t.Fatal(err) - } - if status.Code(newcomer.Grow(t.Context(), 1, Request)) != codes.ResourceExhausted { - t.Fatal("new request overtook reserve borrower") - } - first.Close() - second.Close() - newcomer.Close() - assertEmpty(t, p) -} - -func TestBoundedWaitCancellationAndClose(t *testing.T) { - for _, action := range []string{"cancel", "close", "timeout", "grant-cancel"} { - t.Run(action, func(t *testing.T) { - for range 30 { - p := testPool(t, 10) - p.waitTimeout = 10 * time.Millisecond - held := p.NewOwner().NewLease() - grow(t, held, 9000, Request) - blocked := p.NewOwner().NewLease() - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan error, 1) - go func() { done <- blocked.Grow(ctx, 2000, Response) }() - awaitWaiters(t, p, 1) - switch action { - case "cancel": - cancel() - case "close": - blocked.Close() - case "grant-cancel": - held.Close() - cancel() - blocked.Close() - } - err := <-done - if action != "grant-cancel" && err == nil { - t.Fatal("wait unexpectedly succeeded") - } - cancel() - held.Close() - blocked.Close() - assertEmpty(t, p) - } - }) - } -} - -func TestReserveIsNotFragmented(t *testing.T) { - p := testPool(t, 10) - first, second := p.NewOwner().NewLease(), p.NewOwner().NewLease() - grow(t, first, 4500, Request) - grow(t, second, 4500, Request) - grow(t, first, 400, Response) - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { done <- second.Grow(ctx, 100, Response) }() - awaitWaiters(t, p, 1) - // Remaining reserve belongs to the first owner until it can finish. - grow(t, first, 600, Response) - first.Close() - if err := <-done; err != nil { - t.Fatal(err) - } - second.Close() - assertEmpty(t, p) -} - -func TestImpossibleGrowthAndQueueBound(t *testing.T) { - p := testPool(t, 10) - p.maxWaiters = 1 - held := p.NewOwner().NewLease() - grow(t, held, 9000, Request) - if status.Code(held.Grow(t.Context(), 1001, Response)) != codes.ResourceExhausted { - t.Fatal("impossible request waited") - } - pending := p.NewOwner().NewLease() - done := make(chan error, 1) - go func() { done <- pending.Grow(t.Context(), 2000, Response) }() - awaitWaiters(t, p, 1) - other := p.NewOwner().NewLease() - if status.Code(other.Grow(t.Context(), 2000, Response)) != codes.ResourceExhausted { - t.Fatal("wait queue unbounded") - } - held.Close() - if err := <-done; err != nil { - t.Fatal(err) - } - pending.Close() - other.Close() - assertEmpty(t, p) -} - -func TestScopeRetainsCanceledProducerAndCopies(t *testing.T) { - p := testPool(t, 10) - s := p.NewScope() - if err := s.Admit(t.Context(), 1000); err != nil { - t.Fatal(err) - } - if err := s.Output(t.Context(), 1500); err != nil { - t.Fatal(err) - } - release := s.Retain() - s.Release() - p.mu.Lock() - used := p.used - p.mu.Unlock() - if used != 4000 { - t.Fatalf("premature release or wrong copy charge: %d", used) - } - release() - release() - assertEmpty(t, p) -} - -func TestConcurrentAcquisitionsStayWithinLimit(t *testing.T) { - p := testPool(t, 20) - var wg sync.WaitGroup - for range 32 { - wg.Go(func() { - for range 100 { - l := p.NewOwner().NewLease() - if l.Grow(t.Context(), 100, Request) == nil { - _ = l.Grow(t.Context(), 300, Response) - } - p.mu.Lock() - if p.used < 0 || p.used > p.total { - t.Errorf("invalid use %d", p.used) - } - p.mu.Unlock() - l.Close() - } - }) - } - wg.Wait() - assertEmpty(t, p) -} - -func TestMetricsIncludeIdleZeroes(t *testing.T) { - p := testPool(t, 10) - registry := prometheus.NewPedanticRegistry() - if err := registry.Register(p); err != nil { - t.Fatal(err) - } - if _, err := registry.Gather(); err != nil { - t.Fatal(err) - } - families, err := registry.Gather() - if err != nil { - t.Fatal(err) - } - for _, family := range families { - if family.GetName() == "sink_memory_waiting_bytes" { - if len(family.GetMetric()) != 2 { - t.Fatal("missing idle phases") - } - for _, metric := range family.GetMetric() { - if metric.GetGauge().GetValue() != 0 { - t.Fatal("nonzero idle wait") - } - } - return - } - } - t.Fatal("missing waiting metric") -} - -func TestDetectCapacity(t *testing.T) { - for _, tc := range []struct { - name, membership, mount, filename, value string - soft, want int64 - source string - }{ - {"v2-parent", "0::/pod/child", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/pod/memory.max", "104857600", math.MaxInt64, 50 << 20, "cgroup"}, - {"v1", "1:cpu,memory:/pod", "1 0 0:1 / /sys/fs/cgroup/memory rw - cgroup cgroup rw,memory", "sys/fs/cgroup/memory/pod/memory.limit_in_bytes", "209715200", math.MaxInt64, 100 << 20, "cgroup"}, - {"namespaced", "0::/", "1 0 0:1 /pod /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/memory.max", "104857600", math.MaxInt64, 50 << 20, "cgroup"}, - {"soft", "0::/", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/memory.max", "104857600", 40 << 20, 20 << 20, "gomemlimit"}, - {"unlimited", "0::/", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw", "sys/fs/cgroup/memory.max", "max", math.MaxInt64, 256 << 20, "fallback"}, - } { - t.Run(tc.name, func(t *testing.T) { - files := fstest.MapFS{} - files["proc/self/cgroup"] = &fstest.MapFile{Data: []byte(tc.membership)} - files["proc/self/mountinfo"] = &fstest.MapFile{Data: []byte(tc.mount)} - files[tc.filename] = &fstest.MapFile{Data: []byte(tc.value)} - got, source := detect(files, tc.soft) - if got != tc.want || source != tc.source { - t.Fatalf("got %d %s, want %d %s", got, source, tc.want, tc.source) - } - }) - } -} - -func TestBatchOwnerTransfersOutputLifetimeAndUnusedCredit(t *testing.T) { - p := testPool(t, 10) - producer, caller := p.NewScope(), p.NewScope() - if err := producer.Admit(t.Context(), 4000); err != nil { - t.Fatal(err) - } - if err := caller.Admit(t.Context(), 5000); err != nil { - t.Fatal(err) - } - working := producer.NewLease() - grow(t, working, 500, Response) - if err := caller.OutputFrom(t.Context(), producer, 200); err != nil { - t.Fatal(err) - } - if p.Used() != 9900 { - t.Fatal("batch output did not share the completion owner") - } - producer.Release() - if p.Used() != 5400 { - t.Fatal("producer released another caller's output") - } - if err := caller.EnsureOutput(t.Context(), 200); err != nil { - t.Fatal(err) - } - if p.Used() != 5400 { - t.Fatal("encoding charged transferred credit twice") - } - caller.ReleaseOutputFrom(producer, 100) - if p.Used() != 5200 { - t.Fatal("unused return credit was retained") - } - caller.Release() - assertEmpty(t, p) -} diff --git a/internal/capacity/reader.go b/internal/capacity/reader.go deleted file mode 100644 index f5d57fa..0000000 --- a/internal/capacity/reader.go +++ /dev/null @@ -1,44 +0,0 @@ -package capacity - -import ( - "context" - "io" -) - -// ReadAll acquires each new backing array before allocating it. During growth -// both arrays are charged; after copying only the retained allocation remains. -// The second copy covers JSON decoding while adapters own the response body. -// The caller owns lease and releases it when the body and decoded working set -// are discarded or transferred to separately charged output buffers. -func ReadAll(ctx context.Context, reader io.Reader, maximum int64, lease *Lease) ([]byte, error) { - if lease == nil { - return io.ReadAll(io.LimitReader(reader, maximum)) - } - var body []byte - for { - if len(body) == cap(body) { - if int64(len(body)) == maximum { - return body, nil - } - next := min(maximum, max(1024, int64(cap(body))*2)) - if err := Grow(ctx, lease, int(2*next)); err != nil { - return nil, err - } - grown := make([]byte, len(body), int(next)) - copy(grown, body) - lease.Shrink(2 * int64(cap(body))) - body = grown - } - n, err := reader.Read(body[len(body):cap(body)]) - body = body[:len(body)+n] - if err == io.EOF { - return body, nil - } - if err != nil { - return nil, err - } - if err := ctx.Err(); err != nil { - return nil, err - } - } -} diff --git a/internal/capacity/scope.go b/internal/capacity/scope.go deleted file mode 100644 index d07d03d..0000000 --- a/internal/capacity/scope.go +++ /dev/null @@ -1,178 +0,0 @@ -package capacity - -import ( - "context" - "sync" -) - -type contextKey struct{} - -// Scope follows an RPC through batching and transport ownership. Cancellation -// stops new acquisitions; only the last owner releases already retained bytes. -type Scope struct { - mu sync.Mutex - owner *Owner - refs int - leases []*Lease - input *Lease - output *Lease - outputs map[*Owner]*Lease -} - -func (p *Pool) NewScope() *Scope { - s := &Scope{owner: p.NewOwner(), refs: 1} - s.input = s.NewLease() - s.output = s.NewLease() - return s -} - -func WithScope(ctx context.Context, s *Scope) context.Context { - return context.WithValue(ctx, contextKey{}, s) -} - -func FromContext(ctx context.Context) *Scope { - s, _ := ctx.Value(contextKey{}).(*Scope) - return s -} - -func (s *Scope) NewLease() *Lease { - if s == nil { - return nil - } - s.mu.Lock() - defer s.mu.Unlock() - l := s.owner.NewLease() - if s.refs == 0 { - l.Close() - return l - } - s.leases = append(s.leases, l) - return l -} - -// Retain is called before handing work to another goroutine or transport. -func (s *Scope) Retain() func() { - if s == nil { - return func() {} - } - s.mu.Lock() - if s.refs == 0 { - s.mu.Unlock() - return func() {} - } - s.refs++ - s.mu.Unlock() - return sync.OnceFunc(s.Release) -} - -func (s *Scope) Release() { - if s == nil { - return - } - s.mu.Lock() - defer s.mu.Unlock() - if s.refs == 0 { - return - } - s.refs-- - if s.refs == 0 { - for _, l := range s.leases { - l.Close() - } - s.leases = nil - } -} - -func (s *Scope) Admit(ctx context.Context, bytes int) error { - if s == nil { - return nil - } - return s.input.Grow(ctx, int64(bytes), Request) -} - -// Output acquires a known document and its eventual encoded copy together. -// Reserving the actual encoded size before commit protects returned writes. -func (s *Scope) Output(ctx context.Context, bytes int) error { - return s.OutputFrom(ctx, s, bytes) -} - -// OutputFrom treats batched work as one completion owner; each caller retains its -// own output leases. This avoids a batch blocking on its own borrowed reserve. -func (s *Scope) OutputFrom(ctx context.Context, producer *Scope, bytes int) error { - if s == nil { - return nil - } - if producer == nil || producer.owner == s.owner { - return s.output.Grow(ctx, 2*int64(bytes), Response) - } - s.mu.Lock() - if s.outputs == nil { - s.outputs = make(map[*Owner]*Lease) - } - lease := s.outputs[producer.owner] - if lease == nil { - lease = producer.owner.NewLease() - if s.refs == 0 { - lease.Close() - } else { - s.leases = append(s.leases, lease) - } - s.outputs[producer.owner] = lease - } - s.mu.Unlock() - return lease.Grow(ctx, 2*int64(bytes), Response) -} - -// EnsureOutput reuses document and encoding capacity already acquired by the -// producer. Allocation growth remains atomic in the process pool. -func (s *Scope) EnsureOutput(ctx context.Context, bytes int) error { - if s == nil { - return nil - } - s.mu.Lock() - used := s.output.Bytes() - for _, lease := range s.outputs { - used += lease.Bytes() - } - s.mu.Unlock() - return s.output.Grow(ctx, max(0, 2*int64(bytes)-used), Response) -} - -func Grow(ctx context.Context, lease *Lease, bytes int) error { - if lease == nil { - return nil - } - return lease.Grow(ctx, int64(bytes), Response) -} - -func Close(lease *Lease) { - if lease != nil { - lease.Close() - } -} - -// AdmitCompletion acquires small, known acknowledgement envelopes from ordinary -// capacity before starting work. Only later document growth can use the reserve. -func (s *Scope) AdmitCompletion(ctx context.Context, bytes int) error { - if s == nil { - return nil - } - return s.output.Grow(ctx, 2*int64(bytes), Request) -} - -// ReleaseOutputFrom returns unused pre-commit document capacity after a final -// failure or after a successful candidate shrank. Retained results keep theirs. -func (s *Scope) ReleaseOutputFrom(producer *Scope, bytes int) { - if s == nil || bytes <= 0 { - return - } - s.mu.Lock() - lease := s.output - if producer != nil && producer.owner != s.owner { - lease = s.outputs[producer.owner] - } - s.mu.Unlock() - if lease != nil { - lease.Shrink(2 * int64(bytes)) - } -} diff --git a/internal/config/bytes_test.go b/internal/config/bytes_test.go index 43c9219..b28f0c2 100644 --- a/internal/config/bytes_test.go +++ b/internal/config/bytes_test.go @@ -65,8 +65,6 @@ memory: {max_bytes: 256MiB} producer: {max_buffered_bytes: 64MiB} batching: {max_bytes: 16MiB, queue: {max_bytes: 128MiB}} execution: - max_snapshot_bytes: 16MiB - max_output_bytes: 64MiB merge: lua: {max_source_bytes: 64KiB, max_result_bytes: 16MiB} ` diff --git a/internal/config/config.go b/internal/config/config.go index 9253d1a..1945bb8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,9 +36,9 @@ type Config struct { // Memory is one process-local capacity shared by all request classes. // MaxBytes == 0 selects runtime detection. type Memory struct { - MaxBytes int - BurstPercent int - WaitTimeout time.Duration + MaxBytes int + HighWatermarkPercent int + LowWatermarkPercent int } type GRPC struct { @@ -57,46 +57,16 @@ type Prometheus struct { } type Service struct { - Request Request - Execution Execution - Publish Publish - Batching Batching - Merge Merge + Request Request + Batching Batching + Merge Merge } type Request struct { - Timeout time.Duration MaxOperations int MaxReadBytes int } -type Execution struct { - MaxSnapshotBytes int - MaxOutputBytes int - MongoDB MongoDB - MaxRequests int - MaxBytes int - Queue AdmissionQueue - Scan Scan -} - -type AdmissionQueue struct { - MaxRequests int - MaxBytes int - MaxWait time.Duration -} - -type Scan struct { - MaxRequests int - MaxBytes int - AdmissionWait time.Duration -} - -type Publish struct { - MaxRequests int - MaxBytes int -} - type Batching struct { MaxWait time.Duration MaxOperations int @@ -181,14 +151,11 @@ type Retry struct { // Gateway owns only routing and bounded forwarding resources. type Gateway struct { - MaxRequestsPerStore int - Routes []Route - DNSRefreshInterval time.Duration - IdleTimeout time.Duration - MaxConnections int - MaxRequests int - MaxBytes int - MaxFanout int + Routes []Route + DNSRefreshInterval time.Duration + IdleTimeout time.Duration + MaxConnections int + MaxFanout int } // Route belongs to the Gateway configuration and names one Store's Engines. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4b6cf01..a624686 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,9 +24,9 @@ func writeConfig(t *testing.T, contents string) string { } func TestSharedStoreWithIndependentRoleSettings(t *testing.T) { - storePath := writeConfig(t, kafkaStore) - enginePath := writeConfig(t, "mode: engine\nproducer: {max_buffered_bytes: 8MiB}\nexecution:\n mongodb: {max_concurrent_writes: 17}\n") - workerPath := writeConfig(t, "mode: worker\nconsumer: {group_id: workers, max_poll_records: 2000}\nexecution:\n mongodb: {max_concurrent_writes: 7}\n") + storePath := writeConfig(t, strings.Replace(kafkaStore, " mongodb:\n", " mongodb:\n max_concurrent_writes: 17\n max_concurrent_groups: 5\n", 1)) + enginePath := writeConfig(t, "mode: engine\nproducer: {max_buffered_bytes: 8MiB}\n") + workerPath := writeConfig(t, "mode: worker\nconsumer: {group_id: workers, max_poll_records: 2000}\n") engine, err := Load(enginePath, storePath) if err != nil { t.Fatal(err) @@ -38,7 +38,7 @@ func TestSharedStoreWithIndependentRoleSettings(t *testing.T) { if engine.Storage.Name != worker.Storage.Name || engine.Storage.MongoDB.URI != worker.Storage.MongoDB.URI || !reflect.DeepEqual(engine.Storage.Kafka.Topic, worker.Storage.Kafka.Topic) { t.Fatal("shared identity or dependencies diverged") } - if engine.Storage.MongoDB.MaxConcurrentWrites != 17 || worker.Storage.MongoDB.MaxConcurrentWrites != 7 || worker.Storage.Kafka.Consumer.MaxPollRecords != 2000 || engine.Storage.Kafka.Producer.MaxBufferedBytes != 8<<20 { + if engine.Storage.MongoDB.MaxConcurrentWrites != 17 || worker.Storage.MongoDB.MaxConcurrentWrites != 17 || engine.Storage.MongoDB.MaxConcurrentGroups != 5 || worker.Storage.MongoDB.MaxConcurrentGroups != 5 || worker.Storage.Kafka.Consumer.MaxPollRecords != 2000 || engine.Storage.Kafka.Producer.MaxBufferedBytes != 8<<20 { t.Fatal("role tuning was not independent") } if worker.Storage.Kafka.Consumer.GroupID != "workers" || engine.Storage.Kafka.Consumer.GroupID != "" { @@ -51,20 +51,20 @@ func TestComponentDefaults(t *testing.T) { if err != nil { t.Fatal(err) } - if engine.Service.Request.Timeout != 0 || engine.Service.Request.MaxOperations != 0 || engine.Service.Request.MaxReadBytes != 0 { + if engine.Service.Request.MaxOperations != 0 || engine.Service.Request.MaxReadBytes != 0 { t.Fatal("Engine acquired public request limits") } if engine.Service.Batching.MaxOperations != 32 || engine.Service.Batching.MaxWait != 2*time.Millisecond || engine.Service.Batching.Queue.MaxBytes != 128<<20 { t.Fatal("batch defaults changed") } - if engine.Service.Execution.MaxSnapshotBytes != 32<<20 || engine.Service.Execution.MaxOutputBytes != 32<<20 || engine.Service.Merge.Lua.MaxResultBytes != 16<<20 { + if engine.Storage.MongoDB.MaxConcurrentWrites != 64 || engine.Storage.MongoDB.MaxConcurrentGroups != 16 || engine.Service.Merge.Lua.MaxResultBytes != 16<<20 { t.Fatal("execution defaults changed") } if engine.Storage.Kafka.Enabled || engine.Prometheus.Enabled || engine.Health.Address != ":8081" || engine.GRPC.MaxSendMessageBytes != 64<<20 { t.Fatal("process defaults changed") } gateway, err := Decode(strings.NewReader(gatewayConfig), nil) - if err != nil || gateway.Service.Request.MaxOperations != 1000 || gateway.Service.Request.Timeout != 0 { + if err != nil || gateway.Service.Request.MaxOperations != 1000 { t.Fatalf("Gateway defaults: %v", err) } worker, err := Decode(strings.NewReader("mode: worker\nconsumer: {group_id: workers}\n"), strings.NewReader(kafkaStore)) @@ -88,7 +88,7 @@ func TestStoreIdentityAndBackendValidation(t *testing.T) { "name: primary\nstorage: {driver: opensearch, search: {endpoints: [http://search:9200], username: test}}", "name: primary\nstorage: {driver: opensearch, search: {endpoints: [http://search:9200], username: test, password: test, api_key: test}}", minimalStorage + "consumer: {group_id: workers}\n", minimalStorage + "producer: {}\n", - strings.Replace(minimalStorage, "uri: mongodb://127.0.0.1:1", "uri: mongodb://127.0.0.1:1\n max_concurrent_writes: 3", 1), + strings.Replace(minimalStorage, "uri: mongodb://127.0.0.1:1", "uri: mongodb://127.0.0.1:1\n max_concurrent_writes: 0", 1), } { loaded, err := Decode(strings.NewReader("mode: engine\n"), strings.NewReader(shared)) if err == nil || !reflect.ValueOf(loaded).IsZero() { diff --git a/internal/config/file.go b/internal/config/file.go index 6a8dbbc..6fa8021 100644 --- a/internal/config/file.go +++ b/internal/config/file.go @@ -41,15 +41,7 @@ type requestFile struct { } type executionFile struct { - MaxSnapshotBytes *byteSize `yaml:"max_snapshot_bytes"` - MaxOutputBytes *byteSize `yaml:"max_output_bytes"` - MongoDB mongoExecutionFile `yaml:"mongodb"` - Merge mergeFile `yaml:"merge"` -} - -type mongoExecutionFile struct { - MaxConcurrentWrites *int `yaml:"max_concurrent_writes"` - MaxConcurrentGroups *int `yaml:"max_concurrent_groups"` + Merge mergeFile `yaml:"merge"` } // Store files contain shared identity and dependency policy, never role tuning. @@ -91,9 +83,11 @@ type storageFile struct { } type mongoDBFile struct { - URI *string `yaml:"uri"` - URIFile *string `yaml:"uri_file"` - MetadataField string `yaml:"metadata_field"` + URI *string `yaml:"uri"` + URIFile *string `yaml:"uri_file"` + MetadataField string `yaml:"metadata_field"` + MaxConcurrentWrites *int `yaml:"max_concurrent_writes"` + MaxConcurrentGroups *int `yaml:"max_concurrent_groups"` } type searchFile struct { @@ -149,7 +143,7 @@ type gatewayFile struct { } type memoryFile struct { - MaxBytes *byteSize `yaml:"max_bytes"` - BurstPercent *int `yaml:"burst_percent"` - WaitTimeout *time.Duration `yaml:"wait_timeout"` + MaxBytes *byteSize `yaml:"max_bytes"` + HighWatermarkPercent *int `yaml:"high_watermark_percent"` + LowWatermarkPercent *int `yaml:"low_watermark_percent"` } diff --git a/internal/config/gateway.go b/internal/config/gateway.go index 68cdc43..7e99307 100644 --- a/internal/config/gateway.go +++ b/internal/config/gateway.go @@ -14,9 +14,6 @@ func resolveGateway(file gatewayFile, v *validator) Gateway { loaded.DNSRefreshInterval = v.duration("forwarding.dns_refresh_interval", file.DNSRefreshInterval, 30*time.Second) loaded.IdleTimeout = v.duration("forwarding.idle_timeout", file.IdleTimeout, 5*time.Minute) loaded.MaxConnections = v.bounded("forwarding.max_connections", file.MaxConnections, 256, 10000) - loaded.MaxRequests = v.bounded("forwarding.max_requests", nil, 128, 10000) - loaded.MaxRequestsPerStore = v.bounded("forwarding.max_requests_per_store", nil, min(32, loaded.MaxRequests), loaded.MaxRequests) - loaded.MaxBytes = v.bytes("forwarding.max_bytes", nil, 256<<20, 1<<40) loaded.MaxFanout = v.bounded("forwarding.max_fanout", file.MaxFanout, 8, 128) return loaded } diff --git a/internal/config/limits_test.go b/internal/config/limits_test.go index d096f30..028c067 100644 --- a/internal/config/limits_test.go +++ b/internal/config/limits_test.go @@ -6,16 +6,33 @@ import ( "time" ) -func TestExecutionBudgetsAreIndependentOfTransport(t *testing.T) { - for _, input := range []string{ - "mode: engine\ngrpc: {max_send_message_bytes: 1MiB}\nexecution: {max_snapshot_bytes: 64MiB, max_output_bytes: 128MiB}\n", - "mode: worker\nconsumer: {group_id: workers}\nexecution: {max_snapshot_bytes: 64MiB, max_output_bytes: 128MiB}\n", - } { - loaded, err := Decode(strings.NewReader(input), strings.NewReader(kafkaStore)) - if err != nil || loaded.Service.Execution.MaxSnapshotBytes != 64<<20 || loaded.Service.Execution.MaxOutputBytes != 128<<20 { - t.Fatalf("execution depends on gRPC: %v", err) +func TestExecutionRejectsRemovedLimits(t *testing.T) { + for _, mode := range []string{"engine", "worker"} { + for _, fields := range []string{"max_snapshot_bytes: 64MiB", "max_output_bytes: 128MiB", "mongodb: {max_concurrent_writes: 64}"} { + input := "mode: " + mode + "\nexecution: {" + fields + "}\n" + _, err := Decode(strings.NewReader(input), strings.NewReader(kafkaStore)) + if err == nil || !strings.Contains(err.Error(), "field ") { + t.Fatalf("obsolete execution field accepted: %s: %v", input, err) + } + } + } +} + +func TestMongoDBStoreConcurrencyValidation(t *testing.T) { + for _, field := range []string{"max_concurrent_writes", "max_concurrent_groups"} { + for _, value := range []string{"0", "-1"} { + shared := minimalStorage + " " + field + ": " + value + "\n" + _, err := Decode(strings.NewReader("mode: engine"), strings.NewReader(shared)) + if err == nil || !strings.Contains(err.Error(), "storage.mongodb."+field) { + t.Fatalf("invalid concurrency accepted: %s: %v", shared, err) + } } } + shared := "name: primary\nstorage:\n driver: opensearch\n search: {endpoints: [http://localhost:9200]}\n mongodb: {max_concurrent_writes: 4}\n" + _, err := Decode(strings.NewReader("mode: engine"), strings.NewReader(shared)) + if err == nil || !strings.Contains(err.Error(), "requires the mongodb driver") { + t.Fatalf("MongoDB tuning accepted by another driver: %v", err) + } } func TestRejectInvalidRoleLimits(t *testing.T) { diff --git a/internal/config/load.go b/internal/config/load.go index b7ed240..7965153 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -108,8 +108,11 @@ func resolve(file configFile, shared storeFile) (Config, error) { v.reject(errors.New("memory.max_bytes must be at least 1KiB")) } } - loaded.Memory.BurstPercent = v.bounded("memory.burst_percent", file.Memory.BurstPercent, capacity.DefaultBurstPercent, 99) - loaded.Memory.WaitTimeout = v.duration("memory.wait_timeout", file.Memory.WaitTimeout, 2*time.Second) + loaded.Memory.HighWatermarkPercent = v.bounded("memory.high_watermark_percent", file.Memory.HighWatermarkPercent, capacity.DefaultHighPercent, 99) + loaded.Memory.LowWatermarkPercent = v.bounded("memory.low_watermark_percent", file.Memory.LowWatermarkPercent, capacity.DefaultLowPercent, 99) + if loaded.Memory.LowWatermarkPercent >= loaded.Memory.HighWatermarkPercent { + v.reject(errors.New("memory.low_watermark_percent must be lower than high_watermark_percent")) + } grpcFile := gRPCFile{} if file.GRPC != nil { grpcFile = *file.GRPC @@ -122,9 +125,6 @@ func resolve(file configFile, shared storeFile) (Config, error) { loaded.Prometheus.Address = valueOrDefault(file.Prometheus.Address, ":9090") loaded.ShutdownTimeout = v.duration("shutdown_timeout", file.ShutdownTimeout, 15*time.Second) loaded.Service = resolveService(file, loaded.GRPC, &v) - if loaded.Mode == ModeWorker && (file.Memory.BurstPercent != nil || file.Memory.WaitTimeout != nil) { - v.reject(errors.New("worker memory only accepts max_bytes")) - } if v.err != nil { return loaded, v.err } @@ -159,8 +159,6 @@ func resolve(file configFile, shared storeFile) (Config, error) { if !uri.ValidStore(name) { return loaded, errors.New("store name must be a canonical lowercase identity") } - configured.MongoDB.MaxConcurrentWrites = loaded.Service.Execution.MongoDB.MaxConcurrentWrites - configured.MongoDB.MaxConcurrentGroups = loaded.Service.Execution.MongoDB.MaxConcurrentGroups configured.Kafka = resolveKafka("kafka", shared.Kafka, &v) configured.Kafka.Producer = resolveProducer(file.Producer, &v) configured.Kafka.Consumer = resolveConsumer(file.Consumer, &v) @@ -170,9 +168,6 @@ func resolve(file configFile, shared storeFile) (Config, error) { if loaded.Mode == ModeEngine && configured.Kafka.MaxRecordBytes > configured.Kafka.Producer.MaxBufferedBytes { v.reject(errors.New("producer.max_buffered_bytes must cover kafka.max_record_bytes")) } - if file.Execution != nil && file.Execution.MongoDB != (mongoExecutionFile{}) && configured.Driver != DriverMongoDB { - v.reject(errors.New("execution.mongodb requires the mongodb driver")) - } if v.err != nil { return loaded, v.err } diff --git a/internal/config/memory_test.go b/internal/config/memory_test.go index 76f492f..1a72ec3 100644 --- a/internal/config/memory_test.go +++ b/internal/config/memory_test.go @@ -3,7 +3,6 @@ package config import ( "strings" "testing" - "time" ) func TestMemoryConfiguration(t *testing.T) { @@ -12,18 +11,18 @@ func TestMemoryConfiguration(t *testing.T) { if err != nil { t.Fatal(err) } - if defaults.Memory.MaxBytes != 0 || defaults.Memory.BurstPercent != 10 || defaults.Memory.WaitTimeout != 2*time.Second { + if defaults.Memory.MaxBytes != 0 || defaults.Memory.HighWatermarkPercent != 80 || defaults.Memory.LowWatermarkPercent != 70 { t.Fatalf("defaults: %+v", defaults.Memory) } - explicit := base + "memory: {max_bytes: 128MiB, burst_percent: 15, wait_timeout: 500ms}\n" + explicit := base + "memory: {max_bytes: 128MiB, high_watermark_percent: 90, low_watermark_percent: 75}\n" loaded, err := Decode(strings.NewReader(explicit), nil) if err != nil { t.Fatal(err) } - if loaded.Memory.MaxBytes != 128<<20 || loaded.Memory.BurstPercent != 15 || loaded.Memory.WaitTimeout != 500*time.Millisecond { + if loaded.Memory.MaxBytes != 128<<20 || loaded.Memory.HighWatermarkPercent != 90 || loaded.Memory.LowWatermarkPercent != 75 { t.Fatalf("explicit: %+v", loaded.Memory) } - for _, value := range []string{"burst_percent: 0", "burst_percent: 100", "burst_percent: -1", "max_bytes: 0", "max_bytes: 512", "wait_timeout: 0s", "unknown: 1"} { + for _, value := range []string{"high_watermark_percent: 0", "high_watermark_percent: 100", "high_watermark_percent: -1", "max_bytes: 0", "max_bytes: 512", "low_watermark_percent: 0", "unknown: 1", "high_watermark_percent: 70, low_watermark_percent: 70", "high_watermark_percent: 60", "burst_percent: 10", "wait_timeout: 2s"} { if _, err := Decode(strings.NewReader(base+"memory: {"+value+"}\n"), nil); err == nil { t.Fatalf("accepted %s", value) } diff --git a/internal/config/service.go b/internal/config/service.go index b048c1e..5f83454 100644 --- a/internal/config/service.go +++ b/internal/config/service.go @@ -17,11 +17,6 @@ func resolveService(file configFile, grpc GRPC, v *validator) Service { if execution == nil { execution = &executionFile{} } - loaded.Execution.MaxSnapshotBytes = v.bytes("execution.max_snapshot_bytes", execution.MaxSnapshotBytes, 32<<20, math.MaxInt) - loaded.Execution.MaxOutputBytes = v.bytes("execution.max_output_bytes", execution.MaxOutputBytes, 32<<20, math.MaxInt) - mongo := &loaded.Execution.MongoDB - mongo.MaxConcurrentWrites = v.integer("execution.mongodb.max_concurrent_writes", execution.MongoDB.MaxConcurrentWrites, 64) - mongo.MaxConcurrentGroups = v.integer("execution.mongodb.max_concurrent_groups", execution.MongoDB.MaxConcurrentGroups, 16) loaded.Merge.MaxAttempts = v.integer("execution.merge.max_attempts", execution.Merge.MaxAttempts, 3) lua := &loaded.Merge.Lua lua.Timeout = v.duration("execution.merge.lua.timeout", execution.Merge.Lua.Timeout, 100*time.Millisecond) diff --git a/internal/config/storage.go b/internal/config/storage.go index 90cc57f..42a4038 100644 --- a/internal/config/storage.go +++ b/internal/config/storage.go @@ -9,6 +9,9 @@ func resolveStorage(prefix string, file storageFile) (Storage, error) { var loaded Storage v := validator{} loaded.Driver = Driver(strings.TrimSpace(string(file.Driver))) + if loaded.Driver != DriverMongoDB && file.MongoDB != (mongoDBFile{}) { + return loaded, fmt.Errorf("%s.mongodb requires the mongodb driver", prefix) + } switch loaded.Driver { case DriverMongoDB: mongo := &loaded.MongoDB @@ -21,6 +24,8 @@ func resolveStorage(prefix string, file storageFile) (Storage, error) { return loaded, fmt.Errorf("%s.mongodb.uri is required when driver is mongodb", prefix) } mongo.MetadataField = valueOrDefault(file.MongoDB.MetadataField, "__sink") + mongo.MaxConcurrentWrites = v.integer(prefix+".mongodb.max_concurrent_writes", file.MongoDB.MaxConcurrentWrites, 64) + mongo.MaxConcurrentGroups = v.integer(prefix+".mongodb.max_concurrent_groups", file.MongoDB.MaxConcurrentGroups, 16) case DriverElasticsearch, DriverOpenSearch: search := &loaded.Search search.Endpoints = nonEmptyValues(file.Search.Endpoints) diff --git a/internal/engine/memory_test.go b/internal/engine/memory_test.go new file mode 100644 index 0000000..4b30d27 --- /dev/null +++ b/internal/engine/memory_test.go @@ -0,0 +1,50 @@ +package engine + +import ( + "context" + "os" + "testing" + "testing/fstest" + + forward "github.com/liran/sink/gen/forward" + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/capacity" + "github.com/liran/sink/internal/forwarding" + "google.golang.org/grpc/codes" +) + +type pressureService struct { + sink.UnimplementedSinkServer + calls int +} + +func (s *pressureService) Write(context.Context, *sink.WriteRequest) (*sink.WriteResponse, error) { + s.calls++ + response := &sink.WriteResponse{} + return response, nil +} + +func TestEnginePressureRejectsBeforeExecution(t *testing.T) { + files := fstest.MapFS{"proc/self/statm": {Data: []byte("100 90")}} + memoryOpts := capacity.Options{Bytes: int64(100 * os.Getpagesize()), HighPercent: 80, LowPercent: 70, Files: files} + guard, err := capacity.New(memoryOpts) + if err != nil { + t.Fatal(err) + } + backend := &pressureService{} + opts := Options{Memory: guard, Service: backend, Store: "primary", MaxReadBytes: 1 << 20} + server, err := New(opts) + if err != nil { + t.Fatal(err) + } + write := &sink.WriteRequest{} + body := &forward.ForwardRequest_Write{Write: write} + request := &forward.ForwardRequest{Version: forwarding.Version, Store: "primary", Grant: forwarding.FullBudget(1 << 20), Request: body} + response, err := server.Forward(t.Context(), request) + if err != nil || !response.GetNotStarted() || response.GetCode() != uint32(codes.ResourceExhausted) || backend.calls != 0 { + t.Fatalf("pressure reached service: %v %v calls=%d", response, err, backend.calls) + } + if response.GetUsed().GetReturns() != 0 { + t.Fatal("rejection consumed response allowance") + } +} diff --git a/internal/engine/server.go b/internal/engine/server.go index 74650bd..95bf1b8 100644 --- a/internal/engine/server.go +++ b/internal/engine/server.go @@ -8,6 +8,7 @@ import ( forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/protocol" @@ -16,6 +17,7 @@ import ( ) type Server struct { + memory *capacity.Guard forward.UnimplementedEngineServer service sink.SinkServer store string @@ -24,6 +26,7 @@ type Server struct { metrics *metrics.Metrics } type Options struct { + Memory *capacity.Guard Metrics *metrics.Metrics Service sink.SinkServer Store string @@ -38,7 +41,7 @@ func New(opts Options) (*Server, error) { if opts.MaxRequestBytes == 0 { opts.MaxRequestBytes = 64 << 20 } - server := &Server{maxRequestBytes: opts.MaxRequestBytes, service: opts.Service, store: opts.Store, maximum: opts.MaxReadBytes, metrics: opts.Metrics} + server := &Server{memory: opts.Memory, maxRequestBytes: opts.MaxRequestBytes, service: opts.Service, store: opts.Store, maximum: opts.MaxReadBytes, metrics: opts.Metrics} return server, nil } func (s *Server) Forward(ctx context.Context, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { @@ -87,6 +90,13 @@ func (s *Server) Forward(ctx context.Context, req *forward.ForwardRequest) (*for if err := protocol.CheckStore(request, s.store); err != nil { return reject(err) } + admitted, admissionErr := s.memory.Admit(ctx) + if admissionErr != nil { + return reject(protocol.MemoryAdmissionError(request, admissionErr)) + } + s.metrics.AdjustInFlight(1) + defer s.metrics.AdjustInFlight(-1) + ctx = admitted response.NotStarted = false ctx = forwarding.WithTracker(ctx, tracker) var err error diff --git a/internal/engine/stream.go b/internal/engine/stream.go index a6145c2..fa67970 100644 --- a/internal/engine/stream.go +++ b/internal/engine/stream.go @@ -2,41 +2,19 @@ package engine import ( forward "github.com/liran/sink/gen/forward" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/forwarding" - "github.com/liran/sink/internal/protocol" "google.golang.org/grpc" ) func (s *Server) ForwardStream(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ResponseFrame]) error { ctx := stream.Context() - scope := capacity.FromContext(ctx) - if err := scope.Admit(ctx, 2*req.SizeVT()+1024); err != nil { - return protocol.MemoryAdmissionError(req, err) - } - if err := scope.AdmitCompletion(ctx, protocol.CompletionBytes(req)); err != nil { - return protocol.MemoryAdmissionError(req, err) - } - scratch := scope.NewLease() - if scratch != nil { - if err := scratch.Grow(ctx, forwarding.StreamScratchBytes, capacity.Request); err != nil { - return protocol.MemoryAdmissionError(req, err) - } - } response, err := s.Forward(ctx, req) if err != nil { return err } size := response.SizeVT() - if err := scope.EnsureOutput(ctx, size+1024); err != nil { - return err - } frame := &forward.ResponseFrame{Size: uint64(size)} - var header any = frame - if scope != nil { - header = &protocol.ManagedMessage{Message: frame, Context: ctx} - } - if err := stream.SendMsg(header); err != nil { + if err := stream.SendMsg(frame); err != nil { return err } encoded, err := response.MarshalVT() @@ -45,11 +23,7 @@ func (s *Server) ForwardStream(req *forward.ForwardRequest, stream grpc.ServerSt } for offset := 0; offset < len(encoded); offset += forwarding.FrameBytes { frame := &forward.ResponseFrame{Data: encoded[offset:min(offset+forwarding.FrameBytes, len(encoded))]} - var message any = frame - if scope != nil { - message = &protocol.ManagedMessage{Message: frame, Context: ctx} - } - if err := stream.SendMsg(message); err != nil { + if err := stream.SendMsg(frame); err != nil { return err } } diff --git a/internal/forwarding/budget.go b/internal/forwarding/budget.go index 04d8b39..80e6da4 100644 --- a/internal/forwarding/budget.go +++ b/internal/forwarding/budget.go @@ -9,7 +9,7 @@ import ( "github.com/liran/sink/internal/storage" ) -const Version = 5 +const Version = 6 // EnvelopeBytes is reserved in the private transport in addition to the public message cap. const EnvelopeBytes = 4096 @@ -17,10 +17,7 @@ const EnvelopeBytes = 4096 type Kind int const ( - Snapshots Kind = iota - Inputs - Outputs - Returns + Returns Kind = iota kinds ) @@ -36,7 +33,7 @@ const trackerKey contextKey = 1 func NewTracker(grant *forward.Budget, maximum int) *Tracker { t := &Tracker{} - raw := [...]uint64{grant.GetSnapshots(), grant.GetInputs(), grant.GetOutputs(), grant.GetReturns()} + raw := [...]uint64{grant.GetReturns()} for i, size := range raw { t.limits[i] = int(min(size, uint64(int(^uint(0)>>1)))) if Kind(i) == Returns { @@ -77,11 +74,11 @@ func (t *Tracker) Fresh(kind Kind, maximum int) *storage.ReadBudget { func (t *Tracker) Usage() *forward.Budget { t.mu.Lock() defer t.mu.Unlock() - usage := &forward.Budget{Snapshots: uint64(t.used[Snapshots]), Inputs: uint64(t.used[Inputs]), Outputs: uint64(t.used[Outputs]), Returns: uint64(t.used[Returns])} + usage := &forward.Budget{Returns: uint64(t.used[Returns])} return usage } func FullBudget(maximum int) *forward.Budget { size := uint64(maximum) - budget := &forward.Budget{Snapshots: size, Inputs: size, Outputs: size, Returns: size} + budget := &forward.Budget{Returns: size} return budget } diff --git a/internal/forwarding/frames.go b/internal/forwarding/frames.go index a6daa8b..c919329 100644 --- a/internal/forwarding/frames.go +++ b/internal/forwarding/frames.go @@ -4,6 +4,3 @@ package forwarding // response. Keep the receive limit fixed even when a peer announces a tiny size. const FrameBytes = 32 << 10 const FrameMessageBytes = FrameBytes + 32 - -// StreamScratchBytes covers one fixed HTTP/2 window, framing and decoded chunks. -const StreamScratchBytes = 192 << 10 diff --git a/internal/gateway/batch.go b/internal/gateway/batch.go index b7cce32..8084bc3 100644 --- a/internal/gateway/batch.go +++ b/internal/gateway/batch.go @@ -186,7 +186,7 @@ func parallelBatch(req *forward.ForwardRequest) bool { return true } for _, op := range write.GetOperations() { - if op.GetReturnDocument() || op.GetPut() == nil || op.GetPut().GetMode() != sink.WriteMode_WRITE_MODE_UPSERT { + if op.GetReturnDocument() { return false } } diff --git a/internal/gateway/metrics.go b/internal/gateway/metrics.go index 90dea32..0a67d46 100644 --- a/internal/gateway/metrics.go +++ b/internal/gateway/metrics.go @@ -19,8 +19,6 @@ type metrics struct { requests *prometheus.CounterVec duration *prometheus.HistogramVec inFlight prometheus.Gauge - bytes prometheus.Gauge - rejected prometheus.Counter routes prometheus.Gauge downstream prometheus.Histogram } @@ -28,11 +26,9 @@ type metrics struct { func newMetrics() *metrics { registry := prometheus.NewRegistry() inFlightOpts := prometheus.GaugeOpts{Name: "sink_gateway_in_flight_requests", Help: "Admitted public requests currently forwarding."} - bytesOpts := prometheus.GaugeOpts{Name: "sink_gateway_in_flight_bytes", Help: "Bytes reserved by admitted forwarding requests."} - rejectedOpts := prometheus.CounterOpts{Name: "sink_gateway_rejected_total", Help: "Requests rejected by Gateway admission."} routesOpts := prometheus.GaugeOpts{Name: "sink_gateway_routes", Help: "Number of configured Store routes."} downstreamOpts := prometheus.HistogramOpts{Name: "sink_gateway_engine_duration_seconds", Help: "Engine forwarding latency, including transport failures.", Buckets: prometheus.DefBuckets} - observed := &metrics{registry: registry, inFlight: prometheus.NewGauge(inFlightOpts), bytes: prometheus.NewGauge(bytesOpts), rejected: prometheus.NewCounter(rejectedOpts), routes: prometheus.NewGauge(routesOpts), downstream: prometheus.NewHistogram(downstreamOpts)} + observed := &metrics{registry: registry, inFlight: prometheus.NewGauge(inFlightOpts), routes: prometheus.NewGauge(routesOpts), downstream: prometheus.NewHistogram(downstreamOpts)} configOpts := prometheus.GaugeOpts{Name: "sink_gateway_config_info", Help: "SHA-256 of the normalized routes loaded at startup."} observed.config = prometheus.NewGaugeVec(configOpts, []string{"sha256"}) requestOpts := prometheus.CounterOpts{Name: "sink_gateway_requests_total", Help: "Completed public RPCs."} @@ -40,7 +36,7 @@ func newMetrics() *metrics { durationOpts := prometheus.HistogramOpts{Name: "sink_gateway_request_duration_seconds", Help: "Public RPC latency.", Buckets: prometheus.DefBuckets} observed.duration = prometheus.NewHistogramVec(durationOpts, []string{"method"}) processOpts := collectors.ProcessCollectorOpts{} - registry.MustRegister(observed.config, observed.requests, observed.duration, observed.inFlight, observed.bytes, observed.rejected, observed.routes, observed.downstream, collectors.NewGoCollector(), collectors.NewProcessCollector(processOpts)) + registry.MustRegister(observed.config, observed.requests, observed.duration, observed.inFlight, observed.routes, observed.downstream, collectors.NewGoCollector(), collectors.NewProcessCollector(processOpts)) return observed } func (s *Server) MetricsHandler() http.Handler { diff --git a/internal/gateway/response_budget_test.go b/internal/gateway/response_budget_test.go index 45c2d40..c155c96 100644 --- a/internal/gateway/response_budget_test.go +++ b/internal/gateway/response_budget_test.go @@ -1,10 +1,10 @@ package gateway import ( - "math" "testing" forward "github.com/liran/sink/gen/forward" + sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/config" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -20,12 +20,9 @@ func TestResponseBudgetComesFromTransportAndSharesOnlyReturns(t *testing.T) { if grant.Returns != 4096-2*1280 { t.Fatalf("unreserved response envelope: %v", grant) } - if grant.Snapshots != math.MaxInt64 || grant.Inputs != math.MaxInt64 || grant.Outputs != math.MaxInt64 { - t.Fatalf("transport ceiling leaked into Store-local execution: %v", grant) - } - used := &forward.Budget{Snapshots: 10000, Inputs: 20000, Outputs: 30000, Returns: 100} + used := &forward.Budget{Returns: 100} consume(grant, used) - if grant.Returns != 4096-2*1280-100 || grant.Snapshots != math.MaxInt64 || grant.Outputs != math.MaxInt64 { + if grant.Returns != 4096-2*1280-100 { t.Fatalf("settlement charged unrelated Store working sets: %v", grant) } if _, err := server.responseBudget(4); status.Code(err) != codes.ResourceExhausted { @@ -36,3 +33,26 @@ func TestResponseBudgetComesFromTransportAndSharesOnlyReturns(t *testing.T) { t.Fatalf("native encoded response: %v %v", native, err) } } + +func TestParallelWritesOnlySerializeReturnedDocuments(t *testing.T) { + for _, mode := range []sink.WriteMode{sink.WriteMode_WRITE_MODE_UPSERT, sink.WriteMode_WRITE_MODE_CREATE, sink.WriteMode_WRITE_MODE_REPLACE} { + put := &sink.PutOperation{Mode: mode} + action := &sink.WriteOperation_Put{Put: put} + operation := &sink.WriteOperation{Action: action} + write := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, Operations: []*sink.WriteOperation{operation}} + body := &forward.ForwardRequest_Write{Write: write} + request := &forward.ForwardRequest{Request: body} + if !parallelBatch(request) { + t.Fatal("write without returned documents unnecessarily serialized") + } + merge := &sink.MergeOperation{} + operation.Action = &sink.WriteOperation_Merge{Merge: merge} + if !parallelBatch(request) { + t.Fatal("Merge unnecessarily serialized by removed intermediate budgets") + } + operation.ReturnDocument = true + if parallelBatch(request) { + t.Fatal("returned documents lost their shared response allowance") + } + } +} diff --git a/internal/gateway/server.go b/internal/gateway/server.go index f30ccd1..ca74e95 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -3,8 +3,6 @@ package gateway import ( "context" "errors" - "math" - "sync" "sync/atomic" "time" @@ -13,6 +11,7 @@ import ( "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/config" "github.com/liran/sink/internal/forwarding" + "github.com/liran/sink/internal/protocol" statuspb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -21,19 +20,15 @@ import ( type Server struct { sink.UnimplementedSinkServer config config.Gateway - memory *capacity.Pool + memory *capacity.Guard request config.Request current *snapshot pool connections - mu sync.Mutex - inFlight int - bytes int metrics *metrics - activeStores map[string]int nativeSequence atomic.Uint64 } type Options struct { - Memory *capacity.Pool + Memory *capacity.Guard Gateway config.Gateway Request config.Request MaxMessageBytes int @@ -46,7 +41,7 @@ func New(opts Options) (*Server, error) { } opts.Request.MaxReadBytes = opts.MaxResponseBytes - if opts.Gateway.MaxRequests <= 0 || opts.Gateway.MaxRequestsPerStore <= 0 || opts.Gateway.MaxBytes <= 0 || opts.Gateway.MaxFanout <= 0 || opts.Gateway.MaxConnections <= 0 || opts.Gateway.IdleTimeout <= 0 || opts.Gateway.DNSRefreshInterval <= 0 || opts.Request.MaxOperations <= 0 || opts.MaxMessageBytes <= 0 || opts.MaxResponseBytes <= 0 { + if opts.Gateway.MaxFanout <= 0 || opts.Gateway.MaxConnections <= 0 || opts.Gateway.IdleTimeout <= 0 || opts.Gateway.DNSRefreshInterval <= 0 || opts.Request.MaxOperations <= 0 || opts.MaxMessageBytes <= 0 || opts.MaxResponseBytes <= 0 { return nil, errors.New("gateway limits must be positive") } initial, err := newSnapshot(opts.Gateway.Routes) @@ -57,7 +52,6 @@ func New(opts Options) (*Server, error) { if opts.Memory != nil { server.metrics.registry.MustRegister(opts.Memory) } - server.activeStores = make(map[string]int) server.pool.entries = make(map[Route]*connection) server.pool.maximum = opts.Gateway.MaxConnections server.pool.messageBytes = opts.MaxMessageBytes + forwarding.EnvelopeBytes @@ -84,36 +78,14 @@ func (s *Server) Run(ctx context.Context) { } func (s *Server) Close() { s.pool.close() } func (s *Server) begin(ctx context.Context, req *forward.ForwardRequest) (context.Context, func(), error) { - if s.memory != nil { - return s.beginMemory(ctx, req) - } - // Reserve input, forwarding envelopes and bounded response documents together. - charge := s.reservation(req) - if err := ctx.Err(); err != nil { - return ctx, nil, status.FromContextError(err).Err() - } - s.mu.Lock() - if s.inFlight >= s.config.MaxRequests || charge > s.config.MaxBytes-s.bytes { - s.mu.Unlock() - s.metrics.rejected.Inc() - return ctx, nil, status.Error(codes.ResourceExhausted, "Gateway forwarding capacity is occupied") + admitted, err := s.memory.Admit(ctx) + if err != nil { + return ctx, nil, protocol.MemoryAdmissionError(req, err) } - s.inFlight++ - s.bytes += charge + execution, cancel := context.WithCancel(admitted) s.metrics.inFlight.Inc() - s.metrics.bytes.Add(float64(charge)) - s.mu.Unlock() - ctx, cancel := context.WithCancel(ctx) - release := func() { - cancel() - s.mu.Lock() - s.inFlight-- - s.bytes -= charge - s.metrics.inFlight.Dec() - s.metrics.bytes.Sub(float64(charge)) - s.mu.Unlock() - } - return ctx, release, nil + release := func() { cancel(); s.metrics.inFlight.Dec() } + return execution, release, nil } func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { if route.endpoint == "" { @@ -124,22 +96,6 @@ func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardR defer release() route = targets[(s.nativeSequence.Add(1)-1)%uint64(len(targets))] } - s.mu.Lock() - if s.memory == nil && s.activeStores[route.Store] >= s.config.MaxRequestsPerStore { - s.mu.Unlock() - s.metrics.rejected.Inc() - return localRejection(route, codes.ResourceExhausted, "Store forwarding capacity is occupied"), nil - } - s.activeStores[route.Store]++ - s.mu.Unlock() - defer func() { - s.mu.Lock() - s.activeStores[route.Store]-- - if s.activeStores[route.Store] == 0 { - delete(s.activeStores, route.Store) - } - s.mu.Unlock() - }() req.Version = forwarding.Version req.Store = route.Store entry, err := s.pool.acquire(route) @@ -167,7 +123,7 @@ func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardR return response, nil } func validUsage(grant, used *forward.Budget) bool { - return used.GetSnapshots() <= grant.GetSnapshots() && used.GetInputs() <= grant.GetInputs() && used.GetOutputs() <= grant.GetOutputs() && used.GetReturns() <= grant.GetReturns() + return used.GetReturns() <= grant.GetReturns() } func consume(remaining, used *forward.Budget) { remaining.Returns -= used.GetReturns() @@ -185,49 +141,15 @@ func forwardedError(response *forward.ForwardResponse) error { return status.FromProto(encoded).Err() } -func (s *Server) reservation(req *forward.ForwardRequest) int { - count := len(req.GetRead().GetOperations()) + len(req.GetWrite().GetOperations()) + len(req.GetDelete().GetOperations()) - charge := 2*req.SizeVT() + count*(512+128) + 4096 - documents := req.GetRead() != nil || req.GetExecute() != nil || req.GetQuery() != nil || req.GetScan() != nil - for _, op := range req.GetWrite().GetOperations() { - documents = documents || op.GetReturnDocument() - } - if documents { - charge += 2 * s.request.MaxReadBytes - } - if req.GetCount() != nil { - charge += min(s.request.MaxReadBytes, 256<<10) - } - return charge -} func localRejection(route Route, code codes.Code, message string) *forward.ForwardResponse { usage := &forward.Budget{} response := &forward.ForwardResponse{Version: forwarding.Version, Store: route.Store, Used: usage, Code: uint32(code), Message: message, NotStarted: true} return response } -func (s *Server) beginMemory(ctx context.Context, req *forward.ForwardRequest) (context.Context, func(), error) { - ctx, cancel := context.WithCancel(ctx) - scope := capacity.FromContext(ctx) - release := func() {} - if scope == nil { - scope = s.memory.NewScope() - ctx = capacity.WithScope(ctx, scope) - release = scope.Release - if err := scope.Admit(ctx, 2*req.SizeVT()+1024); err != nil { - cancel() - release() - return ctx, nil, err - } - } - s.metrics.inFlight.Inc() - done := func() { cancel(); release(); s.metrics.inFlight.Dec() } - return ctx, done, nil -} - // responseBudget derives public payload capacity from the transport ceiling. // Result envelopes, revision tokens and bounded failures are reserved before -// forwarding; Store-local snapshots and candidates have independent limits. +// forwarding; Store-local working allocations follow process watermark admission. func (s *Server) responseBudget(operations int) (*forward.Budget, error) { overhead := 0 if operations > 0 { @@ -240,6 +162,6 @@ func (s *Server) responseBudget(operations int) (*forward.Budget, error) { if bytes <= 0 { return nil, status.Error(codes.ResourceExhausted, "response budget is exhausted") } - budget := &forward.Budget{Snapshots: math.MaxInt64, Inputs: math.MaxInt64, Outputs: math.MaxInt64, Returns: uint64(bytes)} + budget := &forward.Budget{Returns: uint64(bytes)} return budget, nil } diff --git a/internal/gateway/stream.go b/internal/gateway/stream.go index 12ef952..6689c2f 100644 --- a/internal/gateway/stream.go +++ b/internal/gateway/stream.go @@ -5,9 +5,7 @@ import ( "io" forward "github.com/liran/sink/gen/forward" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/forwarding" - "github.com/liran/sink/internal/protocol" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -16,20 +14,6 @@ import ( func (s *Server) forwardStream(ctx context.Context, entry *connection, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() - scope := capacity.FromContext(ctx) - scratch := scope.NewLease() - defer capacity.Close(scratch) - if scratch != nil { - if err := scratch.Grow(ctx, forwarding.StreamScratchBytes, capacity.Request); err != nil { - // No request has left this process. Preserve the zero usage settlement - // and safe-retry signal instead of reporting an ambiguous mutation. - marked := protocol.MemoryAdmissionError(req, err) - route := Route{Store: req.GetStore()} - rejected := localRejection(route, status.Code(marked), status.Convert(marked).Message()) - rejected.StatusDetails = status.Convert(marked).Proto().GetDetails() - return rejected, nil - } - } stream, err := entry.client.ForwardStream(ctx, req, grpc.MaxCallRecvMsgSize(forwarding.FrameMessageBytes)) if err != nil { return nil, err @@ -42,11 +26,6 @@ func (s *Server) forwardStream(ctx context.Context, entry *connection, req *forw return nil, status.Error(codes.Internal, "invalid Engine response size frame") } size := int(header.GetSize()) - // The receiver stops reading the stream until capacity exists. A bounded - // frame receive limit and static HTTP/2 window also bound dishonest peers. - if err := scope.Output(ctx, size+1024); err != nil { - return nil, err - } encoded := make([]byte, size) offset := 0 for offset < size { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ba0f859..d025708 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -18,55 +18,43 @@ import ( const namespace = "sink" type Metrics struct { - stores map[string]struct{} - registry *prometheus.Registry - requests *prometheus.CounterVec - requestDuration *prometheus.HistogramVec - operationResults *prometheus.CounterVec - batcherBatches *prometheus.CounterVec - batcherOperations *prometheus.HistogramVec - batcherBytes *prometheus.HistogramVec - batcherQueueDuration *prometheus.HistogramVec - batcherExecution *prometheus.HistogramVec - batcherQueuedOps *prometheus.GaugeVec - batcherQueuedBytes *prometheus.GaugeVec - batcherRejected *prometheus.CounterVec - requestQueueDuration *prometheus.HistogramVec - writePhaseDuration *prometheus.HistogramVec - writeExecutionRounds *prometheus.HistogramVec - requestQueueExits *prometheus.CounterVec - writeSlowPhases *prometheus.CounterVec - mergeConflicts *prometheus.CounterVec - mergeExhausted *prometheus.CounterVec - mergeFoldedChains *prometheus.CounterVec - mergeFoldedOperations *prometheus.CounterVec - kafkaPublished *prometheus.CounterVec - kafkaPublishDuration *prometheus.HistogramVec - kafkaWorkerMutations *prometheus.CounterVec - kafkaWorkerRetries *prometheus.CounterVec - kafkaWorkerDeadLetters *prometheus.CounterVec - admissionRequests prometheus.Gauge - admissionBytes prometheus.Gauge - admissionRejected prometheus.Counter - admissionPoolRequests *prometheus.GaugeVec - admissionPoolBytes *prometheus.GaugeVec - admissionPoolRejected *prometheus.CounterVec - scanQueuedRequests *prometheus.GaugeVec - scanQueuedBytes *prometheus.GaugeVec - scanAdmissionWait *prometheus.HistogramVec - executionQueuedRequests *prometheus.GaugeVec - executionQueuedBytes *prometheus.GaugeVec - executionAdmissionWait *prometheus.HistogramVec - storeExecutionBytes *prometheus.GaugeVec - workerLastPoll *prometheus.GaugeVec - workerLastCommit *prometheus.GaugeVec - workerOldest *prometheus.GaugeVec - workerPending *prometheus.GaugeVec - workerRecoveries *prometheus.CounterVec - workerFetchErrors *prometheus.CounterVec - workerDelivery *prometheus.HistogramVec - workerQuarantined *prometheus.CounterVec - workerOffsetGap *prometheus.GaugeVec + inFlight prometheus.Gauge + stores map[string]struct{} + registry *prometheus.Registry + requests *prometheus.CounterVec + requestDuration *prometheus.HistogramVec + operationResults *prometheus.CounterVec + batcherBatches *prometheus.CounterVec + batcherOperations *prometheus.HistogramVec + batcherBytes *prometheus.HistogramVec + batcherQueueDuration *prometheus.HistogramVec + batcherExecution *prometheus.HistogramVec + batcherQueuedOps *prometheus.GaugeVec + batcherQueuedBytes *prometheus.GaugeVec + batcherRejected *prometheus.CounterVec + requestQueueDuration *prometheus.HistogramVec + writePhaseDuration *prometheus.HistogramVec + writeExecutionRounds *prometheus.HistogramVec + requestQueueExits *prometheus.CounterVec + writeSlowPhases *prometheus.CounterVec + mergeConflicts *prometheus.CounterVec + mergeExhausted *prometheus.CounterVec + mergeFoldedChains *prometheus.CounterVec + mergeFoldedOperations *prometheus.CounterVec + kafkaPublished *prometheus.CounterVec + kafkaPublishDuration *prometheus.HistogramVec + kafkaWorkerMutations *prometheus.CounterVec + kafkaWorkerRetries *prometheus.CounterVec + kafkaWorkerDeadLetters *prometheus.CounterVec + workerLastPoll *prometheus.GaugeVec + workerLastCommit *prometheus.GaugeVec + workerOldest *prometheus.GaugeVec + workerPending *prometheus.GaugeVec + workerRecoveries *prometheus.CounterVec + workerFetchErrors *prometheus.CounterVec + workerDelivery *prometheus.HistogramVec + workerQuarantined *prometheus.CounterVec + workerOffsetGap *prometheus.GaugeVec } type BatchObservation struct { @@ -267,32 +255,8 @@ func New(version string, storeNames ...string) (*Metrics, error) { workerQuarantined := prometheus.NewCounterVec(quarantineOptions, []string{"store"}) registry := prometheus.NewRegistry() registry.MustRegister(workerQuarantined, workerOffsetGap) - admissionRequestsOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "in_flight_requests", Help: "Core requests currently executing, including cross-store and asynchronous requests."} - admissionRequests := prometheus.NewGauge(admissionRequestsOptions) - admissionBytesOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "in_flight_bytes", Help: "Request and output bytes reserved by executing core requests."} - admissionBytes := prometheus.NewGauge(admissionBytesOptions) - admissionRejectedOptions := prometheus.CounterOpts{Namespace: namespace, Name: "admission_rejected_total", Help: "Requests rejected by global or configured-store execution limits."} - admissionRejected := prometheus.NewCounter(admissionRejectedOptions) - poolRequestsOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "admission_pool_requests", Help: "Executing requests in each independently bounded admission pool."} - admissionPoolRequests := prometheus.NewGaugeVec(poolRequestsOptions, []string{"store", "pool"}) - poolBytesOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "admission_pool_bytes", Help: "Bytes reserved in each independently bounded admission pool."} - admissionPoolBytes := prometheus.NewGaugeVec(poolBytesOptions, []string{"store", "pool"}) - poolRejectedOptions := prometheus.CounterOpts{Namespace: namespace, Name: "admission_pool_rejected_total", Help: "Admission rejections by pool and capacity reason."} - admissionPoolRejected := prometheus.NewCounterVec(poolRejectedOptions, []string{"store", "pool", "reason"}) - scanQueueRequestsOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "scan_queued_requests", Help: "Scan pages waiting for execution admission."} - scanQueuedRequests := prometheus.NewGaugeVec(scanQueueRequestsOptions, []string{"store"}) - scanQueueBytesOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "scan_queued_bytes", Help: "Conservative reservation bytes charged to the separate Scan waiting queue."} - scanQueuedBytes := prometheus.NewGaugeVec(scanQueueBytesOptions, []string{"store"}) - scanWaitOptions := prometheus.HistogramOpts{Namespace: namespace, Name: "scan_admission_wait_duration_seconds", Help: "Time queued Scan pages waited before admission, rejection or cancellation.", Buckets: prometheus.DefBuckets} - scanAdmissionWait := prometheus.NewHistogramVec(scanWaitOptions, []string{"store"}) - executionQueueRequestsOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "execution_queued_requests", Help: "Direct synchronous RPCs waiting for execution admission."} - executionQueuedRequests := prometheus.NewGaugeVec(executionQueueRequestsOptions, []string{"store"}) - executionQueueBytesOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "execution_queued_bytes", Help: "Input and bookkeeping bytes charged to the direct synchronous admission queue."} - executionQueuedBytes := prometheus.NewGaugeVec(executionQueueBytesOptions, []string{"store"}) - executionWaitOptions := prometheus.HistogramOpts{Namespace: namespace, Name: "execution_admission_wait_duration_seconds", Help: "Time queued direct synchronous RPCs waited before admission, rejection or cancellation.", Buckets: prometheus.DefBuckets} - executionAdmissionWait := prometheus.NewHistogramVec(executionWaitOptions, []string{"store"}) - storeBytesOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "execution_store_bytes", Help: "Execution reservation bytes attributed to the process Store."} - storeExecutionBytes := prometheus.NewGaugeVec(storeBytesOptions, []string{"store"}) + inFlightOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "in_flight_requests", Help: "Admitted private Engine requests currently queued or executing."} + inFlight := prometheus.NewGauge(inFlightOptions) lastPollOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "kafka_worker_last_poll_timestamp_seconds", Help: "Last completed Kafka poll by configured store."} workerLastPoll := prometheus.NewGaugeVec(lastPollOptions, []string{"store"}) lastCommitOptions := prometheus.GaugeOpts{Namespace: namespace, Name: "kafka_worker_last_commit_timestamp_seconds", Help: "Last successful source offset commit by configured store."} @@ -309,6 +273,7 @@ func New(version string, storeNames ...string) (*Metrics, error) { workerDelivery := prometheus.NewHistogramVec(deliveryOptions, []string{"store"}) processOptions := collectors.ProcessCollectorOpts{} registeredCollectors := []prometheus.Collector{ + inFlight, collectors.NewGoCollector(), collectors.NewProcessCollector(processOptions), buildInfo, @@ -333,13 +298,6 @@ func New(version string, storeNames ...string) (*Metrics, error) { kafkaWorkerMutations, kafkaWorkerRetries, kafkaWorkerDeadLetters, - admissionRequests, - admissionBytes, - admissionRejected, - admissionPoolRequests, admissionPoolBytes, admissionPoolRejected, - scanQueuedRequests, scanQueuedBytes, scanAdmissionWait, - executionQueuedRequests, executionQueuedBytes, executionAdmissionWait, - storeExecutionBytes, workerLastPoll, workerLastCommit, workerOldest, workerPending, workerRecoveries, workerFetchErrors, workerDelivery, } for _, collector := range registeredCollectors { @@ -347,122 +305,46 @@ func New(version string, storeNames ...string) (*Metrics, error) { return nil, fmt.Errorf("register Prometheus collector: %w", err) } } - metrics := &Metrics{ - admissionPoolRequests: admissionPoolRequests, - admissionPoolBytes: admissionPoolBytes, - admissionPoolRejected: admissionPoolRejected, - scanQueuedRequests: scanQueuedRequests, - scanQueuedBytes: scanQueuedBytes, - scanAdmissionWait: scanAdmissionWait, - executionQueuedRequests: executionQueuedRequests, - executionQueuedBytes: executionQueuedBytes, - executionAdmissionWait: executionAdmissionWait, - storeExecutionBytes: storeExecutionBytes, - workerOffsetGap: workerOffsetGap, - workerQuarantined: workerQuarantined, - stores: stores, - registry: registry, - requests: requests, - requestDuration: requestDuration, - operationResults: operationResults, - batcherBatches: batcherBatches, - batcherOperations: batcherOperations, - batcherBytes: batcherBytes, - batcherQueueDuration: batcherQueueDuration, - batcherExecution: batcherExecution, - batcherQueuedOps: batcherQueuedOps, - batcherQueuedBytes: batcherQueuedBytes, - batcherRejected: batcherRejected, - requestQueueDuration: requestQueueDuration, - writePhaseDuration: writePhaseDuration, - writeExecutionRounds: writeExecutionRounds, - requestQueueExits: requestQueueExits, - writeSlowPhases: writeSlowPhases, - mergeConflicts: mergeConflicts, - mergeExhausted: mergeExhausted, - mergeFoldedChains: mergeFoldedChains, - mergeFoldedOperations: mergeFoldedOperations, - kafkaPublished: kafkaPublished, - kafkaPublishDuration: kafkaPublishDuration, - kafkaWorkerMutations: kafkaWorkerMutations, - kafkaWorkerRetries: kafkaWorkerRetries, - kafkaWorkerDeadLetters: kafkaWorkerDeadLetters, - admissionRequests: admissionRequests, - admissionBytes: admissionBytes, - admissionRejected: admissionRejected, - workerLastPoll: workerLastPoll, workerLastCommit: workerLastCommit, workerOldest: workerOldest, + metrics := &Metrics{inFlight: inFlight, + workerOffsetGap: workerOffsetGap, + workerQuarantined: workerQuarantined, + stores: stores, + registry: registry, + requests: requests, + requestDuration: requestDuration, + operationResults: operationResults, + batcherBatches: batcherBatches, + batcherOperations: batcherOperations, + batcherBytes: batcherBytes, + batcherQueueDuration: batcherQueueDuration, + batcherExecution: batcherExecution, + batcherQueuedOps: batcherQueuedOps, + batcherQueuedBytes: batcherQueuedBytes, + batcherRejected: batcherRejected, + requestQueueDuration: requestQueueDuration, + writePhaseDuration: writePhaseDuration, + writeExecutionRounds: writeExecutionRounds, + requestQueueExits: requestQueueExits, + writeSlowPhases: writeSlowPhases, + mergeConflicts: mergeConflicts, + mergeExhausted: mergeExhausted, + mergeFoldedChains: mergeFoldedChains, + mergeFoldedOperations: mergeFoldedOperations, + kafkaPublished: kafkaPublished, + kafkaPublishDuration: kafkaPublishDuration, + kafkaWorkerMutations: kafkaWorkerMutations, + kafkaWorkerRetries: kafkaWorkerRetries, + kafkaWorkerDeadLetters: kafkaWorkerDeadLetters, + workerLastPoll: workerLastPoll, workerLastCommit: workerLastCommit, workerOldest: workerOldest, workerPending: workerPending, workerRecoveries: workerRecoveries, workerFetchErrors: workerFetchErrors, workerDelivery: workerDelivery, } return metrics, nil } -func (m *Metrics) AdjustAdmission(requests int, bytes int) { - if m == nil { - return - } - m.admissionRequests.Add(float64(requests)) - m.admissionBytes.Add(float64(bytes)) -} - -func (m *Metrics) ObserveAdmissionRejected() { - if m == nil { - return - } - m.admissionRejected.Inc() -} - -func (m *Metrics) AdjustAdmissionPool(store string, pool string, requests int, bytes int) { - if m == nil { - return - } - m.AdjustAdmission(requests, bytes) - m.admissionPoolRequests.WithLabelValues(m.storeLabel(store), pool).Add(float64(requests)) - m.admissionPoolBytes.WithLabelValues(m.storeLabel(store), pool).Add(float64(bytes)) -} - -func (m *Metrics) ObserveAdmissionPoolRejected(store string, pool string, reason string) { - if m == nil { - return - } - m.ObserveAdmissionRejected() - m.admissionPoolRejected.WithLabelValues(m.storeLabel(store), pool, reason).Inc() -} - -func (m *Metrics) AdjustScanQueue(store string, requests int, bytes int) { - if m == nil { - return - } - m.scanQueuedRequests.WithLabelValues(m.storeLabel(store)).Add(float64(requests)) - m.scanQueuedBytes.WithLabelValues(m.storeLabel(store)).Add(float64(bytes)) -} - -func (m *Metrics) ObserveScanAdmissionWait(store string, duration time.Duration) { - if m == nil { - return - } - m.scanAdmissionWait.WithLabelValues(m.storeLabel(store)).Observe(duration.Seconds()) -} - -func (m *Metrics) AdjustExecutionQueue(store string, requests int, bytes int) { - if m == nil { - return - } - m.executionQueuedRequests.WithLabelValues(m.storeLabel(store)).Add(float64(requests)) - m.executionQueuedBytes.WithLabelValues(m.storeLabel(store)).Add(float64(bytes)) -} - -func (m *Metrics) ObserveExecutionAdmissionWait(store string, duration time.Duration) { - if m == nil { - return - } - m.executionAdmissionWait.WithLabelValues(m.storeLabel(store)).Observe(duration.Seconds()) -} - -func (m *Metrics) AdjustStoreExecutionBytes(store string, bytes int) { - if m == nil { - return +func (m *Metrics) AdjustInFlight(requests int) { + if m != nil { + m.inFlight.Add(float64(requests)) } - m.storeExecutionBytes.WithLabelValues(m.storeLabel(store)).Add(float64(bytes)) } func (m *Metrics) ObserveWorkerPoll(store string, errors int) { diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 6f285b0..9ed394d 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -95,21 +95,16 @@ func TestMetricsExposeBuildRequestAndOperationResults(t *testing.T) { observed.ObserveMergeFold("primary", 1) observed.ObserveMergeFold("primary", 16) observed.ObserveMergeFold("primary", 4) - observed.AdjustAdmissionPool("primary", "execution", 1, 100) - observed.AdjustAdmissionPool("primary", "publish", 1, 20) - observed.AdjustAdmissionPool("primary", "publish", -1, -20) - observed.ObserveAdmissionPoolRejected("primary", "execution", "fairness") - observed.ObserveAdmissionPoolRejected("primary", "publish", "bytes") + observed.AdjustInFlight(1) body := scrape(t, observed) + for _, obsolete := range []string{"sink_in_flight_bytes", "sink_admission_", "sink_scan_queued_", "sink_scan_admission_", "sink_execution_queued_", "sink_execution_admission_", "sink_execution_store_bytes"} { + if strings.Contains(body, obsolete) { + t.Errorf("obsolete metric exported: %s", obsolete) + } + } wanted := []string{ `sink_in_flight_requests 1`, - `sink_in_flight_bytes 100`, - `sink_admission_rejected_total 2`, - `sink_admission_pool_requests{pool="execution",store="primary"} 1`, - `sink_admission_pool_bytes{pool="publish",store="primary"} 0`, - `sink_admission_pool_rejected_total{pool="execution",reason="fairness",store="primary"} 1`, - `sink_admission_pool_rejected_total{pool="publish",reason="bytes",store="primary"} 1`, `sink_build_info{version="test-version"} 1`, `sink_grpc_server_requests_total{code="OK",method="Read",store="_unconfigured"} 1`, `sink_grpc_server_operation_results_total{method="Read",status="found",store="_unconfigured"} 1`, diff --git a/internal/metrics/store_test.go b/internal/metrics/store_test.go index 1ff021c..96a757d 100644 --- a/internal/metrics/store_test.go +++ b/internal/metrics/store_test.go @@ -187,11 +187,6 @@ func TestStoreClassificationBoundsUnknownEmptyAndNilRequests(t *testing.T) { observed.ObserveMergeConflict(store, 1) observed.ObserveMergeFold(store, 2) observed.ObserveMergeExhausted(store, 1) - observed.AdjustAdmissionPool(store, "execution", 1, 1) - observed.ObserveAdmissionPoolRejected(store, "execution", "bytes") - observed.AdjustScanQueue(store, 1, 10) - observed.ObserveScanAdmissionWait(store, time.Second) - observed.AdjustStoreExecutionBytes(store, 10) observed.AdjustBatchQueue(store, "Write", 1, 1) observed.ObserveBatchRejected(store, "Write", "queue_full") batch := sinkmetrics.BatchObservation{Store: store, Method: "Write", Reason: "max_wait"} diff --git a/internal/metrics/write.go b/internal/metrics/write.go index 2d4cd4f..10aac45 100644 --- a/internal/metrics/write.go +++ b/internal/metrics/write.go @@ -50,7 +50,7 @@ func (m *Metrics) ObserveWritePhase(observation WritePhaseObservation) { } phase := observation.Phase switch phase { - case "admission", "parse", "storage_read", "lua": + case "parse", "storage_read", "lua": case "storage_write": if observation.Completion == sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE { phase = "storage_write_visible" diff --git a/internal/metrics/write_cardinality_test.go b/internal/metrics/write_cardinality_test.go index ab7630d..90e9bea 100644 --- a/internal/metrics/write_cardinality_test.go +++ b/internal/metrics/write_cardinality_test.go @@ -35,7 +35,7 @@ func TestWriteDiagnosticsSeriesBudget(t *testing.T) { names = append(names, "_multiple", "_unconfigured") for _, store := range names { for _, mode := range []sink.CompletionMode{sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE} { - for _, phase := range []string{"admission", "parse", "storage_read", "lua", "storage_write"} { + for _, phase := range []string{"parse", "storage_read", "lua", "storage_write"} { observation := sinkmetrics.WritePhaseObservation{Store: store, Completion: mode, Phase: phase, Duration: 12 * time.Second} observed.ObserveWritePhase(observation) } @@ -74,9 +74,9 @@ func TestWriteDiagnosticsSeriesBudget(t *testing.T) { wanted := map[string]int{ "sink_batcher_request_queue_duration_seconds": 30 * stores, "sink_batcher_request_queue_exits_total": 9 * stores, - "sink_write_phase_duration_seconds": 60 * (stores + 2), + "sink_write_phase_duration_seconds": 50 * (stores + 2), "sink_write_execution_rounds": 18 * (stores + 2), - "sink_write_slow_phases_total": 6 * (stores + 2), + "sink_write_slow_phases_total": 5 * (stores + 2), } total := 0 for name, count := range counts { @@ -85,7 +85,7 @@ func TestWriteDiagnosticsSeriesBudget(t *testing.T) { t.Errorf("%s: %s has %d series, budget %d", accept, name, count, wanted[name]) } } - if len(counts) != len(wanted) || total != 123*stores+168 || strings.Contains(body, "unbounded-") { + if len(counts) != len(wanted) || total != 112*stores+146 || strings.Contains(body, "unbounded-") { t.Fatalf("%s: unexpected series cardinality: %v, total %d", accept, counts, total) } t.Logf("%s: %d configured stores, %d added series per Pod", accept, stores, total) diff --git a/internal/protocol/grpc_codec.go b/internal/protocol/grpc_codec.go index 8b5c6b0..a72b098 100644 --- a/internal/protocol/grpc_codec.go +++ b/internal/protocol/grpc_codec.go @@ -3,8 +3,6 @@ package protocol import ( "fmt" - "github.com/liran/sink/internal/capacity" - "google.golang.org/grpc/encoding" _ "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/mem" @@ -39,9 +37,6 @@ func (*VTProtoCodec) Name() string { } func (c *VTProtoCodec) Marshal(value any) (mem.BufferSlice, error) { - if managed, ok := value.(*ManagedMessage); ok { - return c.marshalManaged(managed) - } message, ok := value.(vtProtoCodecMessage) if !ok { if c.fallback == nil { @@ -91,39 +86,3 @@ func (c *VTProtoCodec) Unmarshal(data mem.BufferSlice, value any) error { defer buffer.Free() return message.UnmarshalVT(buffer.ReadOnlyData()) } - -// The pool callback runs on the last grpc/mem reference, including references -// held by HTTP/2 after SendMsg or the unary handler has returned. -type ownedBufferPool struct{ release func() } - -func (*ownedBufferPool) Get(size int) *[]byte { data := make([]byte, size); return &data } -func (p *ownedBufferPool) Put(*[]byte) { p.release() } - -func (c *VTProtoCodec) marshalManaged(managed *ManagedMessage) (mem.BufferSlice, error) { - message, ok := managed.Message.(vtProtoCodecMessage) - if !ok { - return c.Marshal(managed.Message) - } - scope := capacity.FromContext(managed.Context) - size := message.SizeVT() - // grpc/mem's small SliceBuffer has no final-free callback. Allocate just - // above its threshold so even tiny replies retain their ownership correctly. - allocation := max(size, 1025) - if err := scope.EnsureOutput(managed.Context, allocation); err != nil { - return nil, err - } - release := scope.Retain() - buffer := make([]byte, size, allocation) - written, err := message.MarshalToSizedBufferVT(buffer) - if err != nil { - release() - return nil, err - } - if written != size { - release() - return nil, fmt.Errorf("vtproto: marshaled %d bytes, expected %d", written, size) - } - pool := &ownedBufferPool{release: release} - encoded := mem.BufferSlice{mem.NewBuffer(&buffer, pool)} - return encoded, nil -} diff --git a/internal/protocol/memory.go b/internal/protocol/memory.go index 2b83355..2780c0b 100644 --- a/internal/protocol/memory.go +++ b/internal/protocol/memory.go @@ -1,94 +1,15 @@ package protocol import ( - "context" "errors" - "strings" forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/capacity" "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc" - "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) -// MemoryStats owns the RPC reference until gRPC finishes it. Batch producers -// and encoded transport buffers keep independent references past cancellation. -type MemoryStats struct { - Pool *capacity.Pool -} - -func (h *MemoryStats) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context { - if strings.HasPrefix(info.FullMethodName, "/sink.") { - return capacity.WithScope(ctx, h.Pool.NewScope()) - } - return ctx -} - -func (*MemoryStats) HandleRPC(ctx context.Context, event stats.RPCStats) { - if _, ok := event.(*stats.End); ok { - capacity.FromContext(ctx).Release() - - } -} -func (*MemoryStats) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { return ctx } -func (*MemoryStats) HandleConn(context.Context, stats.ConnStats) {} - -// ManagedMessage carries memory ownership only inside the local gRPC stack; -// its protobuf representation and the public API are unchanged. -type ManagedMessage struct { - Message any - Context context.Context -} - -func MemoryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - scope := capacity.FromContext(ctx) - if scope == nil { - return handler(ctx, req) - } - if sized, ok := req.(interface{ SizeVT() int }); ok { - if err := scope.Admit(ctx, 2*sized.SizeVT()+384*operationCount(req)+1024); err != nil { - return nil, MemoryAdmissionError(req, err) - } - } - if err := scope.AdmitCompletion(ctx, CompletionBytes(req)); err != nil { - return nil, MemoryAdmissionError(req, err) - } - response, err := handler(ctx, req) - if err != nil { - return nil, err - } - if sized, ok := response.(interface{ SizeVT() int }); ok { - if err := scope.EnsureOutput(ctx, sized.SizeVT()+1024); err != nil { - return nil, err - } - wrapped := &ManagedMessage{Message: response, Context: ctx} - return wrapped, nil - } - return response, nil -} - -func operationCount(message any) int { - switch req := message.(type) { - case *sink.ReadRequest: - return len(req.GetOperations()) - case *sink.WriteRequest: - return len(req.GetOperations()) - case *sink.DeleteRequest: - return len(req.GetOperations()) - case *forward.ForwardRequest: - return len(req.GetRead().GetOperations()) + len(req.GetWrite().GetOperations()) + len(req.GetDelete().GetOperations()) - default: - return 0 - } -} - -// CompletionBytes covers bounded per-operation failures and protobuf envelopes. -// Document bytes are acquired once their actual sizes become known. -func CompletionBytes(message any) int { return 4096 + 1280*operationCount(message) } - // MemoryAdmissionError preserves the Scan SDK's safe retry signal, exclusively // for temporary refusal before invoking the service or forwarding downstream. func MemoryAdmissionError(message any, err error) error { diff --git a/internal/protocol/memory_test.go b/internal/protocol/memory_test.go index 0d7dd43..2c89b7e 100644 --- a/internal/protocol/memory_test.go +++ b/internal/protocol/memory_test.go @@ -1,89 +1,30 @@ package protocol_test import ( - "bytes" - "context" - sink "github.com/liran/sink/gen/sink" - "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "testing" - "time" + sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/protocol" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -func TestManagedCodecReleasesOnLastTransportReference(t *testing.T) { - for _, size := range []int{0, 1, 1024, 1025, 65536} { - t.Run(string(rune('a'+size%26)), func(t *testing.T) { - opts := capacity.Options{Bytes: 1 << 20, BurstPercent: 10, WaitTimeout: time.Second} - p, err := capacity.New(opts) - if err != nil { - t.Fatal(err) - } - scope := p.NewScope() - ctx := capacity.WithScope(t.Context(), scope) - if err := scope.Admit(ctx, 1024); err != nil { - t.Fatal(err) - } - message := &trackedVTMessage{data: bytes.Repeat([]byte{42}, size)} - wrapped := &protocol.ManagedMessage{Message: message, Context: ctx} - encoded, err := protocol.NewVTProtoCodec().Marshal(wrapped) - if err != nil { - t.Fatal(err) - } - encoded.Ref() - encoded.Free() - scope.Release() // gRPC stats.End may precede the transport's final Free. - if p.Used() == 0 { - t.Fatal("transport still owns an unaccounted buffer") - } - if !bytes.Equal(encoded.Materialize(), message.data) { - t.Fatal("invalid encoded message") - } - encoded.Free() - if p.Used() != 0 { - t.Fatalf("transport leaked %d bytes", p.Used()) - } - }) - } -} - func TestMemoryAdmissionPreservesSafeScanRetryOnlyBeforeExecution(t *testing.T) { - opts := capacity.Options{Bytes: 1 << 20, BurstPercent: 10, WaitTimeout: time.Second} - pool, err := capacity.New(opts) - if err != nil { - t.Fatal(err) - } - blocker := pool.NewOwner().NewLease() - if err := blocker.Grow(t.Context(), 943000, capacity.Request); err != nil { - t.Fatal(err) - } - defer blocker.Close() - scope := pool.NewScope() - defer scope.Release() - ctx := capacity.WithScope(t.Context(), scope) - req := &sink.ScanRequest{} - info := &grpc.UnaryServerInfo{FullMethod: "/sink.v1.Sink/Scan"} - called := false - handler := func(context.Context, any) (any, error) { called = true; return nil, nil } - _, err = protocol.MemoryInterceptor(ctx, req, info, handler) - if called || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("admission result: called=%v err=%v", called, err) - } + scan := &sink.ScanRequest{} + err := protocol.MemoryAdmissionError(scan, capacity.ErrBusy) details := status.Convert(err).Details() if len(details) != 1 { - t.Fatalf("missing Scan retry detail: %v", details) + t.Fatal("missing pre-execution retry detail") } detail, ok := details[0].(*errdetails.ErrorInfo) if !ok || detail.GetReason() != "SCAN_ADMISSION_REJECTED" { - t.Fatalf("wrong retry detail: %v", details) + t.Fatal(details) } - oversize := status.Error(codes.ResourceExhausted, "allocation exceeds process memory capacity") - if len(status.Convert(protocol.MemoryAdmissionError(req, oversize)).Details()) != 0 { - t.Fatal("permanent failure advertised safe retry") + backendError := status.Error(codes.ResourceExhausted, "backend resource exhausted") + if len(status.Convert(protocol.MemoryAdmissionError(scan, backendError)).Details()) != 0 { + t.Fatal("backend failure advertised safe retry") } write := &sink.WriteRequest{} if len(status.Convert(protocol.MemoryAdmissionError(write, capacity.ErrBusy)).Details()) != 0 { diff --git a/internal/queue/kafka/capacity_split_test.go b/internal/queue/kafka/capacity_split_test.go index 5b05a9e..85f6049 100644 --- a/internal/queue/kafka/capacity_split_test.go +++ b/internal/queue/kafka/capacity_split_test.go @@ -2,12 +2,18 @@ package kafka import ( "context" + "fmt" + "io/fs" + "os" "strings" + "sync/atomic" "testing" + "testing/fstest" "time" "github.com/liran/sink-go/uri" sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/merge" "github.com/liran/sink/internal/queue" "github.com/liran/sink/internal/service" @@ -33,7 +39,7 @@ func (h *capacitySplitHandler) HandleBatch(ctx context.Context, mutations []queu return results } -func TestWorkerSplitsCapacityRejectedPollAndCommits(t *testing.T) { +func TestWorkerPausesForMemoryPressureAndCommitsCollectedPoll(t *testing.T) { cluster, err := kfake.NewCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, "review-capacity", "review-capacity.dlq")) if err != nil { t.Fatal(err) @@ -45,8 +51,8 @@ func TestWorkerSplitsCapacityRejectedPollAndCommits(t *testing.T) { t.Fatal(err) } store := memory.New() - // One 600-byte mutation plus its failure response fits; two do not. - coreOpts := service.Options{BoundStore: "primary", Storage: store, Lua: engine, MaxInFlightBytes: 2048} + // The admitted poll is processed whole after memory pressure recovers. + coreOpts := service.Options{BoundStore: "primary", Storage: store, Lua: engine} core, err := service.New(coreOpts) if err != nil { t.Fatal(err) @@ -56,7 +62,14 @@ func TestWorkerSplitsCapacityRejectedPollAndCommits(t *testing.T) { t.Fatal(err) } handler := &capacitySplitHandler{processor: processor, calls: make(chan int, 100)} - opts := WorkerOptions{Brokers: cluster.ListenAddrs(), Store: "primary", Topic: "review-capacity", GroupID: "review-capacity", DeadLetterTopic: "review-capacity.dlq", Handler: handler, MaxPollRecords: 2, MaxRetryAttempts: 1, RetryBackoff: time.Millisecond, MaxRetryBackoff: time.Millisecond} + pressure := &pressureCounter{} + pressure.pages.Store(90) + memoryOpts := capacity.Options{Bytes: int64(100 * os.Getpagesize()), HighPercent: 80, LowPercent: 70, Files: pressure} + guard, err := capacity.New(memoryOpts) + if err != nil { + t.Fatal(err) + } + opts := WorkerOptions{Memory: guard, Brokers: cluster.ListenAddrs(), Store: "primary", Topic: "review-capacity", GroupID: "review-capacity", DeadLetterTopic: "review-capacity.dlq", Handler: handler, MaxPollRecords: 2, MaxRetryAttempts: 1, RetryBackoff: time.Millisecond, MaxRetryBackoff: time.Millisecond} w, err := NewWorker(opts) if err != nil { t.Fatal(err) @@ -82,6 +95,15 @@ func TestWorkerSplitsCapacityRejectedPollAndCommits(t *testing.T) { done := make(chan error, 1) go func() { done <- w.Run(ctx) }() defer func() { cancel(); <-done }() + // No handler call or offset commit may occur while intake is paused. + select { + case <-handler.calls: + t.Fatal("Worker polled above its high watermark") + case <-time.After(2 * capacity.SampleInterval): + } + // Replace the filesystem only while no concurrent counter read can observe it. + // The fake counter below is backed by an atomic value for subsequent samples. + pressure.pages.Store(60) select { case size := <-handler.calls: if size != 2 { @@ -103,3 +125,11 @@ func TestWorkerSplitsCapacityRejectedPollAndCommits(t *testing.T) { t.Fatal("valid records were quarantined") } } + +// pressureCounter supplies changing OS readings without a test-only function hook. +type pressureCounter struct{ pages atomic.Int64 } + +func (p *pressureCounter) Open(name string) (fs.File, error) { + files := fstest.MapFS{"proc/self/statm": {Data: []byte(fmt.Sprintf("100 %d", p.pages.Load()))}} + return files.Open(name) +} diff --git a/internal/queue/kafka/worker.go b/internal/queue/kafka/worker.go index 1e9e855..0b8387c 100644 --- a/internal/queue/kafka/worker.go +++ b/internal/queue/kafka/worker.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/logging" sinkmetrics "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/queue" @@ -36,6 +37,7 @@ type Handler interface { } type WorkerOptions struct { + Memory *capacity.Guard Topics *TopicManager Brokers []string Store string @@ -55,6 +57,8 @@ type WorkerOptions struct { } type Worker struct { + memory *capacity.Guard + topic string topics *TopicManager client *kgo.Client handler Handler @@ -126,7 +130,7 @@ func NewWorker(opts WorkerOptions) (*Worker, error) { if opts.ShutdownTimeout == 0 { opts.ShutdownTimeout = 5 * time.Second } - worker := &Worker{ + worker := &Worker{memory: opts.Memory, topic: opts.Topic, topics: opts.Topics, handler: opts.Handler, store: opts.Store, @@ -187,6 +191,13 @@ func (w *Worker) Run(ctx context.Context) error { } fetchBackoff := w.retryBackoff for { + if w.memory.Blocked() { + w.client.PauseFetchTopics(w.topic) + if err := w.memory.Wait(ctx); err != nil { + return nil + } + w.client.ResumeFetchTopics(w.topic) + } fetches := w.client.PollRecords(ctx, w.maxPollRecords) if ctx.Err() != nil { w.client.AllowRebalance() diff --git a/internal/service/admission.go b/internal/service/admission.go deleted file mode 100644 index fa531cb..0000000 --- a/internal/service/admission.go +++ /dev/null @@ -1,475 +0,0 @@ -package service - -import ( - "context" - "slices" - "sync" - "time" - - "github.com/liran/sink/internal/capacity" - "github.com/liran/sink/internal/protocol" - - sink "github.com/liran/sink/gen/sink" - sinkmetrics "github.com/liran/sink/internal/metrics" - "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// A zero timeout preserves the caller's deadline without imposing a server limit. -func executionContext(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { - if timeout == 0 { - return context.WithCancel(ctx) - } - return context.WithTimeout(ctx, timeout) -} - -// Publishing has its own bounded pool so storage latency and snapshot waiters -// cannot consume the capacity needed to durably enqueue asynchronous work. -type admissionPool struct { - name string - metrics *sinkmetrics.Metrics - requestTimeout time.Duration - maxInFlightRequests int - maxInFlightBytes int - admissionMu sync.Mutex - admissionWaiters []*admissionRequest - inFlightRequests int - inFlightBytes int - maxScanRequests int - maxScanBytes int - scanRequests int - scanBytes int - scanAdmissionWait time.Duration - maxQueuedRequests int - maxQueuedBytes int - admissionWait time.Duration - queuedRequests int - queuedBytes int -} - -type admissionRequest struct { - encodedBytes int - resultBytes int - sharedInput bool - stores []string - wait bool - timeout time.Duration - scan bool - publish bool - reservation *admissionReservation - inputBytes int - direct bool - ready chan struct{} -} - -func (s *Server) admitRequest(ctx context.Context, request admissionRequest) (context.Context, context.CancelFunc, error) { - if s.memory != nil { - return s.admitMemory(ctx, request) - } - if request.publish { - return s.publishAdmission.admitRequest(ctx, request) - } - if !request.wait && !request.scan && request.inputBytes > 0 { - request.wait = true - request.direct = true - // Charge decoded input and per-call bookkeeping, not hypothetical - // response buffers that are only allocated after execution admission. - request.inputBytes += 256 - } - return s.admissionPool.admitRequest(ctx, request) -} - -func (s *admissionPool) admitRequest(ctx context.Context, request admissionRequest) (context.Context, context.CancelFunc, error) { - started := time.Now() - store := s.metrics.RequestStores(request.stores) - waitCtx := ctx - if request.scan && request.wait { - var cancel context.CancelFunc - waitCtx, cancel = context.WithTimeout(ctx, s.scanAdmissionWait) - defer cancel() - } - if request.direct { - var cancel context.CancelFunc - waitCtx, cancel = context.WithTimeout(ctx, s.admissionWait) - defer cancel() - } - var queued *admissionRequest - var scanQueuedAt time.Time - var directQueuedAt time.Time - defer func() { - if queued != nil { - s.admissionMu.Lock() - s.removeAdmissionWaiter(queued) - s.admissionMu.Unlock() - } - if !scanQueuedAt.IsZero() { - s.metrics.ObserveScanAdmissionWait(store, time.Since(scanQueuedAt)) - } - if !directQueuedAt.IsZero() { - s.metrics.ObserveExecutionAdmissionWait(store, time.Since(directQueuedAt)) - } - }() - for { - if err := contextError(ctx); err != nil { - return ctx, nil, err - } - if queued != nil && waitCtx.Err() != nil { - if err := contextError(ctx); err != nil { - return ctx, nil, err - } - return ctx, nil, s.rejectAdmission(request, "wait_timeout") - } - s.admissionMu.Lock() - full := s.admissionSlotsFull(request) || request.encodedBytes > s.maxInFlightBytes-s.inFlightBytes - reason := "requests" - if request.encodedBytes > s.maxInFlightBytes-s.inFlightBytes { - reason = "bytes" - } - for _, earlier := range s.admissionWaiters { - if earlier == queued { - break - } - // A runnable older batch reserves the next available byte capacity. - // Otherwise small arrivals can indefinitely starve returned-document - // batches. A busy scan pool must still let ordinary requests run. - if !s.admissionSlotsFull(*earlier) { - if !full { - reason = "fairness" - } - full = true - break - } - } - if full { - canWait := request.wait && s.admissionCanFit(request) - if canWait && queued == nil { - if request.direct && s.directWaitQueueFull(request) { - s.admissionMu.Unlock() - return ctx, nil, s.rejectAdmission(request, "queue") - } - if request.scan && s.scanWaitQueueFull(request) { - s.admissionMu.Unlock() - return ctx, nil, s.rejectAdmission(request, "queue") - } - request.ready = make(chan struct{}, 1) - queued = &request - s.admissionWaiters = append(s.admissionWaiters, queued) - if request.direct { - directQueuedAt = time.Now() - s.adjustDirectQueue(request, 1) - } - if request.scan { - scanQueuedAt = time.Now() - s.metrics.AdjustScanQueue(store, 1, request.encodedBytes) - } - } - changed := request.ready - s.admissionMu.Unlock() - if canWait { - select { - case <-changed: - continue - case <-waitCtx.Done(): - if err := contextError(ctx); err != nil { - return ctx, nil, err - } - return ctx, nil, s.rejectAdmission(request, "wait_timeout") - } - } - return ctx, nil, s.rejectAdmission(request, reason) - } - s.inFlightRequests++ - s.inFlightBytes += request.encodedBytes - s.adjustStoreBytes(request.stores, request.encodedBytes) - if request.reservation != nil { - request.reservation.pool = s - request.reservation.stores = request.stores - request.reservation.bytes = request.encodedBytes - request.reservation.maximum = request.encodedBytes - } - if request.scan { - s.scanRequests++ - s.scanBytes += request.encodedBytes - } - if queued != nil { - s.removeAdmissionWaiter(queued) - queued = nil - } - s.admissionMu.Unlock() - break - } - s.metrics.AdjustAdmissionPool(store, s.name, 1, request.encodedBytes) - timeout := request.timeout - if timeout == 0 { - timeout = s.requestTimeout - } - if request.direct && timeout > 0 { - timeout -= time.Since(started) - } - execution, cancel := executionContext(ctx, timeout) - release := func() { - cancel() - s.admissionMu.Lock() - bytes := request.encodedBytes - if request.reservation != nil { - bytes = request.reservation.bytes - request.reservation.released = true - } - s.inFlightRequests-- - s.inFlightBytes -= bytes - s.adjustStoreBytes(request.stores, -bytes) - if request.scan { - s.scanRequests-- - s.scanBytes -= request.encodedBytes - } - s.wakeAdmissionWaiter() - s.admissionMu.Unlock() - s.metrics.AdjustAdmissionPool(store, s.name, -1, -bytes) - } - return execution, release, nil -} - -func (s *admissionPool) admissionCanFit(request admissionRequest) bool { - return request.encodedBytes <= s.maxInFlightBytes && (!request.scan || request.encodedBytes <= s.maxScanBytes) -} - -// The caller holds admissionMu. The execution pool belongs to one Store. -func (s *admissionPool) adjustStoreBytes(stores []string, bytes int) { - if s.name == "execution" { - s.metrics.AdjustStoreExecutionBytes(s.metrics.RequestStores(stores), bytes) - } -} - -// The caller holds admissionMu. Queued scans have separate count and byte -// bounds equal to the execution scan sublimits. Charge the full -// reservation conservatively, including input, without taking execution slots. -func (s *admissionPool) scanWaitQueueFull(request admissionRequest) bool { - requests, bytes := 0, 0 - for _, queued := range s.admissionWaiters { - if !queued.scan { - continue - } - requests++ - bytes += queued.encodedBytes - } - return requests >= s.maxScanRequests || request.encodedBytes > s.maxScanBytes-bytes -} - -func (s *admissionPool) rejectAdmission(request admissionRequest, reason string) error { - store := s.metrics.RequestStores(request.stores) - s.metrics.ObserveAdmissionPoolRejected(store, s.name, reason) - rejection := status.New(codes.ResourceExhausted, "Sink "+s.name+" capacity is full") - if request.scan && s.admissionCanFit(request) { - // This detail is emitted only before backend execution and only for - // temporary pressure. Oversized requests and backend/page failures must - // never acquire this retry signal. - detail := &errdetails.ErrorInfo{Domain: "sink", Reason: "SCAN_ADMISSION_REJECTED", - Metadata: map[string]string{"pool": s.name, "reason": reason}} - if detailed, err := rejection.WithDetails(detail); err == nil { - rejection = detailed - } - } - return rejection.Err() -} - -// The caller holds admissionMu. Global bytes are handled separately so a -// waiting large request can accumulate space without reserving a store slot. -func (s *admissionPool) admissionSlotsFull(request admissionRequest) bool { - if s.inFlightRequests >= s.maxInFlightRequests { - return true - } - if request.scan && (s.scanRequests >= s.maxScanRequests || request.encodedBytes > s.maxScanBytes-s.scanBytes) { - return true - } - return false -} - -func (s *admissionPool) removeAdmissionWaiter(request *admissionRequest) { - index := slices.Index(s.admissionWaiters, request) - if index < 0 { - return - } - s.admissionWaiters = slices.Delete(s.admissionWaiters, index, index+1) - if request.direct { - s.adjustDirectQueue(*request, -1) - } - if request.scan { - store := s.metrics.RequestStores(request.stores) - s.metrics.AdjustScanQueue(store, -1, -request.encodedBytes) - } - s.wakeAdmissionWaiter() -} - -// The caller holds admissionMu. Hand capacity to the oldest eligible waiter; -// admission or cancellation of that waiter wakes the next one. Broadcasting -// every release makes all queued RPCs contend for the same lock and CPU quota. -func (s *admissionPool) wakeAdmissionWaiter() { - for _, request := range s.admissionWaiters { - if s.admissionSlotsFull(*request) { - continue - } - if request.encodedBytes <= s.maxInFlightBytes-s.inFlightBytes { - select { - case request.ready <- struct{}{}: - default: - } - } - // Reserve accumulating global bytes for this older eligible request. - return - } -} - -func operationStores[T interface{ GetAddress() *sink.RecordAddress }](operations []T) []string { - seen := make(map[string]struct{}) - for _, operation := range operations { - seen[protocol.RecordStore(operation.GetAddress())] = struct{}{} - } - stores := make([]string, 0, len(seen)) - for name := range seen { - stores = append(stores, name) - } - return stores -} - -// Include retained output and expanded source copies, before parsing or cloning. -// This bounds admitted payload bytes; VM/driver overhead is sized separately. -type writeExecutionEstimate struct { - bytes int - workingBytes int -} - -func (s *Server) estimateWriteExecution(req *sink.WriteRequest, callers int, returningCallers int) writeExecutionEstimate { - estimate := writeExecutionEstimate{} - if s.memory != nil { - estimate.bytes = 2*req.SizeVT() + failureResponseBytes(len(req.GetOperations())) - return estimate - } - bytes := req.SizeVT() + failureResponseBytes(len(req.GetOperations())) - bytes += s.estimateWriteReturns(req, returningCallers) - largestSource := 0 - for _, program := range req.GetLuaPrograms() { - largestSource = max(largestSource, len(program.GetSource())) - } - hasSnapshot := false - hasConditionalPut := false - if len(req.GetOperations()) > 1 { - for _, operation := range req.GetOperations() { - if operation.GetPut() != nil && operation.GetPut().GetMode() != sink.WriteMode_WRITE_MODE_UPSERT { - hasConditionalPut = true - break - } - } - } - type putRun struct { - count int - conditional bool - } - puts := make(map[recordIdentity]putRun) - for _, operation := range req.GetOperations() { - if operation.GetMerge() != nil { - hasSnapshot = true - bytes += max(largestSource, len(operation.GetMerge().GetLuaProgram().GetSource())) - } - if hasConditionalPut && !hasSnapshot && operation.GetPut() != nil { - address, err := protocol.ParseAddress(operation.GetAddress()) - if err == nil { - key := identityOf(address) - run := puts[key] - run.count++ - run.conditional = run.conditional || operation.GetPut().GetMode() != sink.WriteMode_WRITE_MODE_UPSERT - puts[key] = run - hasSnapshot = run.count > 1 && run.conditional - } - } - } - if hasSnapshot && req.GetCompletionMode() != sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED { - // Shared records retain one snapshot and final document, even when - // several RPCs contribute to the chain. Preserve hot-key folding while - // reserving separate space for independent callers/records. - retained := callers - if callers > 1 { - records := make(map[recordIdentity]bool) - for _, operation := range req.GetOperations() { - address, err := protocol.ParseAddress(operation.GetAddress()) - if err == nil { - records[identityOf(address)] = true - } - } - retained = min(callers, max(1, len(records))) - } - // A micro-batch streams independent records through one bounded read - // chunk and one output batch. One additional candidate can coexist with - // the output batch while it is committed. Caller quotas remain separate. - workingSets := min(2*retained, 3) - estimate.workingBytes = s.maxReadBytes * workingSets - bytes += estimate.workingBytes - } - estimate.bytes = bytes - return estimate -} - -func (s *Server) estimateWriteReturns(req *sink.WriteRequest, callers int) int { - maximum := s.maxReadBytes * callers - bytes := 0 - for _, operation := range req.GetOperations() { - if !operation.GetReturnDocument() { - continue - } - if operation.GetMerge() != nil { - // Lua output is unknown until execution; retain the original RPC - // allowance for every returning caller in a mixed batch. - return maximum - } - // Returned Put documents are known before execution. Every returned - // operation owns a copy, including repeated writes of the same key. - bytes += len(operation.GetPut().GetDocument().GetPayload()) + 128 - } - return min(bytes, maximum) -} - -func returningCallerCount(req *sink.WriteRequest, budgets *requestBudgets) int { - owners := make(map[int]bool) - for index, operation := range req.GetOperations() { - if operation.GetReturnDocument() { - owners[budgets.owner(index)] = true - } - } - return len(owners) -} - -// The application always supplies the process pool. The old admission fields -// remain for explicit legacy-limit regression and benchmark fixtures. -func (s *Server) admitMemory(ctx context.Context, request admissionRequest) (context.Context, context.CancelFunc, error) { - execution, cancel := executionContext(ctx, s.requestTimeout) - scope := capacity.FromContext(execution) - releaseScope := func() {} - if scope == nil { - scope = s.memory.NewScope() - execution = capacity.WithScope(execution, scope) - releaseScope = scope.Release - bytes := 2*request.inputBytes + 2*request.resultBytes + 4096 - if request.sharedInput { - // Coalescing aliases original request messages. Their RPC scopes already - // own those bytes; acquire only batch bookkeeping and result structures. - bytes = request.resultBytes + 4096 - } - if err := scope.Admit(execution, bytes); err != nil { - cancel() - releaseScope() - return ctx, nil, err - } - } - if request.reservation != nil { - request.reservation.managed = scope.NewLease() - request.reservation.ctx = execution - } - release := func() { - cancel() - if request.reservation != nil { - capacity.Close(request.reservation.managed) - } - releaseScope() - } - return execution, release, nil -} diff --git a/internal/service/admission_cancellation_test.go b/internal/service/admission_cancellation_test.go deleted file mode 100644 index 6428122..0000000 --- a/internal/service/admission_cancellation_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package service - -import ( - "context" - "sync" - "testing" - "testing/synctest" - - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// Expire immediately after the second Err snapshot. This places cancellation -// between the parent check and the wait-context check after an admission wakeup. -type admissionExpiringContext struct { - context.Context - mu sync.Mutex - done chan struct{} - err error - expired bool - checks int -} - -func (c *admissionExpiringContext) Done() <-chan struct{} { return c.done } - -func (c *admissionExpiringContext) Err() error { - c.mu.Lock() - defer c.mu.Unlock() - if c.expired { - return c.err - } - c.checks++ - if c.checks == 2 { - c.expired = true - close(c.done) - } - return nil -} - -func TestAdmissionPreservesCancellationAfterWakeup(t *testing.T) { - for _, cause := range []error{context.Canceled, context.DeadlineExceeded} { - t.Run(cause.Error(), func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - pool := completionServer(t, memory.New()).server.admissionPool - pool.maxInFlightBytes = 10 - busy := admissionRequest{encodedBytes: 3} - _, releaseBusy, err := pool.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - other := admissionRequest{encodedBytes: 1} - _, releaseOther, err := pool.admitRequest(t.Context(), other) - if err != nil { - t.Fatal(err) - } - defer releaseOther() - ctx := &admissionExpiringContext{Context: t.Context(), done: make(chan struct{}), err: cause} - waiting := admissionRequest{encodedBytes: 8, wait: true} - finished := make(chan error, 1) - go func() { - _, release, err := pool.admitRequest(ctx, waiting) - if release != nil { - release() - } - finished <- err - }() - synctest.Wait() - releaseBusy() - if err := <-finished; status.Code(err) != status.FromContextError(cause).Code() { - t.Fatalf("caller cancellation became capacity rejection: %v", err) - } - if pool.inFlightBytes != 1 || pool.inFlightRequests != 1 || len(pool.admissionWaiters) != 0 { - t.Fatal("canceled waiter leaked admission state") - } - followup := admissionRequest{encodedBytes: 9} - _, release, err := pool.admitRequest(t.Context(), followup) - if status.Code(err) != codes.OK { - t.Fatalf("canceled waiter blocked available capacity: %v", err) - } - release() - }) - }) - } -} diff --git a/internal/service/admission_fairness_test.go b/internal/service/admission_fairness_test.go deleted file mode 100644 index e84246c..0000000 --- a/internal/service/admission_fairness_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package service - -import ( - "context" - "testing" - "testing/synctest" - - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestAdmissionPreservesCapacityForWaitingLargeBatch(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := completionServer(t, memory.New()).server - server.maxInFlightBytes = 100 - initial := admissionRequest{encodedBytes: 20} - _, releaseInitial, err := server.admitRequest(t.Context(), initial) - if err != nil { - t.Fatal(err) - } - large := admissionRequest{encodedBytes: 90, wait: true} - admitted := make(chan context.CancelFunc, 1) - go func() { - _, release, admitErr := server.admitRequest(t.Context(), large) - if admitErr != nil { - t.Error(admitErr) - } - admitted <- release - }() - synctest.Wait() - // An unlimited stream of these smaller requests must not keep the - // older batch waiting until its caller deadline. - small := admissionRequest{encodedBytes: 1} - _, releaseSmall, err := server.admitRequest(t.Context(), small) - if releaseSmall != nil { - releaseSmall() - } - if status.Code(err) != codes.ResourceExhausted { - t.Errorf("new request bypassed an older byte reservation: %v", err) - } - releaseInitial() - synctest.Wait() - releaseLarge := <-admitted - if releaseLarge != nil { - releaseLarge() - } - }) -} - -func TestAdmissionCancellationWakesNextReservation(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := completionServer(t, memory.New()).server - server.maxInFlightBytes = 100 - initial := admissionRequest{encodedBytes: 20} - _, releaseInitial, err := server.admitRequest(t.Context(), initial) - if err != nil { - t.Fatal(err) - } - defer releaseInitial() - ctx, cancel := context.WithCancel(t.Context()) - large := admissionRequest{encodedBytes: 90, wait: true} - canceled := make(chan error, 1) - go func() { - _, _, admitErr := server.admitRequest(ctx, large) - canceled <- admitErr - }() - synctest.Wait() - small := admissionRequest{encodedBytes: 1, wait: true} - admitted := make(chan context.CancelFunc, 1) - go func() { - _, release, admitErr := server.admitRequest(t.Context(), small) - if admitErr != nil { - t.Error(admitErr) - } - admitted <- release - }() - synctest.Wait() - if len(admitted) != 0 { - t.Error("later waiter bypassed the large batch") - } - cancel() - synctest.Wait() - if status.Code(<-canceled) != codes.Canceled { - t.Error("canceled reservation did not return cancellation") - } - releaseSmall := <-admitted - if releaseSmall != nil { - releaseSmall() - } - if len(server.admissionWaiters) != 0 || server.inFlightBytes != 20 { - t.Fatal("completed or canceled reservation leaked admission state") - } - }) -} diff --git a/internal/service/admission_queue.go b/internal/service/admission_queue.go deleted file mode 100644 index 18c345f..0000000 --- a/internal/service/admission_queue.go +++ /dev/null @@ -1,17 +0,0 @@ -package service - -// The caller holds admissionMu. Direct RPCs share one bounded input queue; -// batched RPCs and scans retain their existing separate queue bounds. -func (s *admissionPool) directWaitQueueFull(request admissionRequest) bool { - if s.queuedRequests >= s.maxQueuedRequests || request.inputBytes > s.maxQueuedBytes-s.queuedBytes { - return true - } - return false -} - -func (s *admissionPool) adjustDirectQueue(request admissionRequest, delta int) { - s.queuedRequests += delta - s.queuedBytes += delta * request.inputBytes - store := s.metrics.RequestStores(request.stores) - s.metrics.AdjustExecutionQueue(store, delta, delta*request.inputBytes) -} diff --git a/internal/service/admission_queue_test.go b/internal/service/admission_queue_test.go deleted file mode 100644 index b9f713c..0000000 --- a/internal/service/admission_queue_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package service - -import ( - "context" - "testing" - "testing/synctest" - "time" - - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestDirectAdmissionQueuesBurst(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - core := completionServer(t, memory.New()).server - core.maxInFlightRequests = 1 - request := admissionRequest{encodedBytes: 64 << 20, inputBytes: 100, stores: []string{"a"}} - _, release, err := core.admitRequest(t.Context(), request) - if err != nil { - t.Fatal(err) - } - admitted := make(chan context.CancelFunc, 1) - go func() { - _, done, err := core.admitRequest(t.Context(), request) - if err != nil { - t.Error(err) - } - admitted <- done - }() - synctest.Wait() - if len(admitted) != 0 || core.queuedBytes >= 1024 || core.queuedRequests != 1 || core.inFlightRequests != 1 { - t.Fatal("waiting request executed or reserved hypothetical output buffers") - } - release() - done := <-admitted - if done != nil { - done() - } - if core.queuedRequests != 0 || core.queuedBytes != 0 || core.inFlightBytes != 0 { - t.Fatal("admitted burst leaked queue or execution capacity") - } - }) -} - -func TestDirectAdmissionBoundsQueuedCountAndBytes(t *testing.T) { - for _, bound := range []string{"requests", "bytes"} { - t.Run(bound, func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - core := completionServer(t, memory.New()).server - core.maxInFlightRequests = 1 - switch bound { - case "requests": - core.maxQueuedRequests = 1 - case "bytes": - core.maxQueuedBytes = 500 - } - request := admissionRequest{encodedBytes: 1, inputBytes: 100, stores: []string{"primary"}} - _, release, err := core.admitRequest(t.Context(), request) - if err != nil { - t.Fatal(err) - } - defer release() - ctx, cancel := context.WithCancel(t.Context()) - finished := make(chan error, 1) - go func() { _, _, err := core.admitRequest(ctx, request); finished <- err }() - synctest.Wait() - _, done, err := core.admitRequest(t.Context(), request) - if done != nil { - done() - } - if status.Code(err) != codes.ResourceExhausted || core.queuedRequests != 1 { - t.Fatalf("unbounded admission queue: %v", err) - } - cancel() - if err := <-finished; status.Code(err) != codes.Canceled { - t.Fatalf("cancellation became a resource failure: %v", err) - } - if core.queuedBytes != 0 || core.queuedRequests != 0 || len(core.admissionWaiters) != 0 { - t.Fatal("canceled request retained queue capacity") - } - }) - }) - } -} - -func TestDirectAdmissionWaitDeadlineAndImpossibleRequests(t *testing.T) { - for _, callerDeadline := range []bool{false, true} { - synctest.Test(t, func(t *testing.T) { - core := completionServer(t, memory.New()).server - core.maxInFlightBytes = 100 - busy := admissionRequest{encodedBytes: 100} - _, release, err := core.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - defer release() - request := admissionRequest{encodedBytes: 101, inputBytes: 1} - start := time.Now() - _, _, err = core.admitRequest(t.Context(), request) - if status.Code(err) != codes.ResourceExhausted || time.Since(start) != 0 { - t.Fatal("impossible request occupied the waiting queue") - } - request.encodedBytes = 50 - ctx := t.Context() - want := codes.ResourceExhausted - if callerDeadline { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Millisecond) - defer cancel() - want = codes.DeadlineExceeded - } - _, _, err = core.admitRequest(ctx, request) - if status.Code(err) != want || core.queuedRequests != 0 || core.inFlightBytes != 100 { - t.Fatalf("deadline outcome or accounting: %v", err) - } - }) - } -} - -func TestAdmissionHandoffFillsEveryAvailableSlot(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - core := completionServer(t, memory.New()).server - core.maxInFlightBytes = 100 - busy := admissionRequest{encodedBytes: 100} - _, release, err := core.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - admitted := make(chan context.CancelFunc, 16) - for range 16 { - go func() { - request := admissionRequest{encodedBytes: 1, inputBytes: 1} - _, done, err := core.admitRequest(t.Context(), request) - if err != nil { - t.Error(err) - } - admitted <- done - }() - } - synctest.Wait() - release() - synctest.Wait() - if len(admitted) != 16 || core.queuedRequests != 0 { - t.Fatal("handoff stranded runnable waiters without another completion") - } - for range 16 { - if done := <-admitted; done != nil { - done() - } - } - }) -} - -func TestDirectAdmissionWaitConsumesRequestDeadline(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - core := completionServer(t, memory.New()).server - core.requestTimeout = time.Second - core.admissionWait = time.Second - core.maxInFlightRequests = 1 - busy := admissionRequest{encodedBytes: 1} - _, release, err := core.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - started := time.Now() - go func() { time.Sleep(500 * time.Millisecond); release() }() - request := admissionRequest{encodedBytes: 1, inputBytes: 1} - execution, done, err := core.admitRequest(t.Context(), request) - if err != nil { - t.Fatal(err) - } - defer done() - deadline, ok := execution.Deadline() - if !ok || !deadline.Equal(started.Add(time.Second)) { - t.Fatalf("admission reset the request timeout: %v", deadline) - } - }) -} diff --git a/internal/service/admission_test.go b/internal/service/admission_test.go deleted file mode 100644 index dee56eb..0000000 --- a/internal/service/admission_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package service - -import ( - "context" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/liran/sink-go/uri" - "github.com/liran/sink/internal/testuri" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/merge" - sinkmetrics "github.com/liran/sink/internal/metrics" - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type blockedReadStorage struct { - storage.Storage - started chan struct{} -} - -func (b *blockedReadStorage) Read(ctx context.Context, req storage.ReadRequest) (storage.ReadResponse, error) { - b.started <- struct{}{} - <-ctx.Done() - var response storage.ReadResponse - return response, ctx.Err() -} - -func TestCoreAdmissionBoundsRequestsAndReleasesCancellation(t *testing.T) { - observed, err := sinkmetrics.New("test", "a") - if err != nil { - t.Fatal(err) - } - luaOptions := merge.LuaOptions{} - lua, err := merge.NewLuaEngine(luaOptions) - if err != nil { - t.Fatal(err) - } - backend := &blockedReadStorage{Storage: memory.New(), started: make(chan struct{}, 4)} - opts := Options{BoundStore: "a", Storage: backend, Lua: lua, Metrics: observed, MaxInFlightRequests: 1, MaxReadBytes: 1024, RequestTimeout: 100 * time.Millisecond, AdmissionWait: 10 * time.Millisecond} - s, err := New(opts) - if err != nil { - t.Fatal(err) - } - a := admissionRead("a") - b := admissionRead("a") - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan struct{}) - go func() { _, _ = s.Read(ctx, a); close(done) }() - <-backend.started - cross := &sink.ReadRequest{Operations: []*sink.ReadOperation{a.Operations[0], b.Operations[0]}} - if _, err := s.Read(t.Context(), cross); status.Code(err) != codes.ResourceExhausted { - t.Fatalf("cross-store call bypassed occupied store limit: %v", err) - } - request := httptest.NewRequest("GET", "/metrics", nil) - recorder := httptest.NewRecorder() - observed.Handler().ServeHTTP(recorder, request) - body := recorder.Body.String() - for _, line := range []string{ - `sink_admission_pool_requests{pool="execution",store="a"} 1`, - `sink_admission_pool_rejected_total{pool="execution",reason="wait_timeout",store="a"} 1`, - `sink_execution_queued_requests{store="a"} 0`, - `sink_execution_queued_bytes{store="a"} 0`, - `sink_in_flight_requests 1`, - `sink_admission_rejected_total 1`, - } { - if !strings.Contains(body, line) { - t.Errorf("missing metric: %s", line) - } - } - cancel() - <-done - // A caller without a deadline is still released by the server's deadline. - _, err = s.Read(t.Context(), b) - if err == nil { - t.Fatal("blocked read unexpectedly succeeded") - } - if s.inFlightRequests != 0 || s.inFlightBytes != 0 { - t.Fatal("cancelled/timed-out calls leaked admission capacity") - } -} - -func admissionRead(store string) *sink.ReadRequest { - kind := uri.StringKey("key") - key := kind - address := &sink.RecordAddress{Uri: testuri.Record(store, []string{"n", "d"}, key)} - op := &sink.ReadOperation{Address: address} - req := &sink.ReadRequest{Operations: []*sink.ReadOperation{op}} - return req -} - -func TestExecutionContextSurvivesOneCallerButCancelsForAll(t *testing.T) { - first, cancelFirst := context.WithCancel(t.Context()) - second, cancelSecond := context.WithCancel(t.Context()) - defer cancelFirst() - defer cancelSecond() - calls := []*batchCall[int, int]{{ctx: first}, {ctx: second}} - ctx, cleanup := batchExecutionContext(t.Context(), calls, time.Second) - defer cleanup() - cancelFirst() - select { - case <-ctx.Done(): - t.Fatal("one caller cancelled the other caller's work") - case <-time.After(20 * time.Millisecond): - } - cancelSecond() - select { - case <-ctx.Done(): - case <-time.After(time.Second): - t.Fatal("all cancelled callers retained execution") - } -} - -func TestFoldedConditionalPutsReserveSnapshotCapacity(t *testing.T) { - for _, conditionalFirst := range []bool{false, true} { - backend := &blockedReadStorage{Storage: memory.New(), started: make(chan struct{}, 1)} - server := completionServer(t, backend).server - server.maxReadBytes = 256 - first := completionPut("hot", 1) - second := completionPut("hot", 2) - if conditionalFirst { - first.GetPut().Mode = sink.WriteMode_WRITE_MODE_CREATE - } else { - second.GetPut().Mode = sink.WriteMode_WRITE_MODE_REPLACE - } - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, Operations: []*sink.WriteOperation{first, second}} - want := request.SizeVT() + failureResponseBytes(len(request.Operations)) + 2*server.maxReadBytes - if got := server.estimateWriteExecution(request, 1, returningCallerCount(request, nil)).bytes; got != want { - t.Fatalf("conditional first=%t: admission bytes=%d want=%d", conditionalFirst, got, want) - } - server.maxInFlightBytes = want - 1 - _, err := server.Write(t.Context(), request) - if status.Code(err) != codes.ResourceExhausted || len(backend.started) != 0 { - t.Fatalf("folded puts read before reserving snapshot bytes: %v", err) - } - } -} - -func (b *blockedReadStorage) BatchKey(address storage.Address) (string, error) { - return testuri.BatchKey(address) -} diff --git a/internal/service/batch_context_test.go b/internal/service/batch_context_test.go new file mode 100644 index 0000000..a7f8917 --- /dev/null +++ b/internal/service/batch_context_test.go @@ -0,0 +1,29 @@ +package service + +import ( + "context" + "testing" + "time" +) + +func TestExecutionContextSurvivesOneCallerButCancelsForAll(t *testing.T) { + first, cancelFirst := context.WithCancel(t.Context()) + second, cancelSecond := context.WithCancel(t.Context()) + defer cancelFirst() + defer cancelSecond() + calls := []*batchCall[int, int]{{ctx: first}, {ctx: second}} + ctx, cleanup := batchExecutionContext(t.Context(), calls, time.Second) + defer cleanup() + cancelFirst() + select { + case <-ctx.Done(): + t.Fatal("one caller cancelled the other caller's work") + case <-time.After(20 * time.Millisecond): + } + cancelSecond() + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("all cancelled callers retained execution") + } +} diff --git a/internal/service/batcher.go b/internal/service/batcher.go index 1adbf26..e30c5f3 100644 --- a/internal/service/batcher.go +++ b/internal/service/batcher.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/liran/sink/internal/capacity" sinkmetrics "github.com/liran/sink/internal/metrics" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -19,8 +18,6 @@ type batchResult[Response any] struct { type batchCall[Request any, Response any] struct { ctx context.Context - releaseMemory func() - executing bool request Request operationCount int encodedBytes int @@ -128,7 +125,6 @@ func (b *requestBatcher[Request, Response]) Submit( call := &batchCall[Request, Response]{ ctx: ctx, - releaseMemory: capacity.FromContext(ctx).Retain(), request: request, operationCount: operationCount, encodedBytes: encodedBytes, @@ -150,7 +146,6 @@ func (b *requestBatcher[Request, Response]) Submit( select { case <-b.ctx.Done(): b.release(call) - call.releaseMemory() return empty, status.Error(codes.Unavailable, "synchronous batcher is shutting down") default: } @@ -158,11 +153,9 @@ func (b *requestBatcher[Request, Response]) Submit( case b.input <- call: case <-ctx.Done(): b.release(call) - call.releaseMemory() return empty, contextError(ctx) case <-b.ctx.Done(): b.release(call) - call.releaseMemory() return empty, status.Error(codes.Unavailable, "synchronous batcher is shutting down") } @@ -276,7 +269,6 @@ func (b *requestBatcher[Request, Response]) run() { if reason != "max_wait" || wait <= 0 { pending = remaining for _, call := range selected { - call.executing = true b.release(call) call.pendingRecords = make(map[recordIdentity]bool, len(call.records)) for _, key := range call.records { @@ -383,13 +375,6 @@ func (b *requestBatcher[Request, Response]) executeBatch( calls []*batchCall[Request, Response], reason string, ) { - defer func() { - for _, call := range calls { - if call.releaseMemory != nil { - call.releaseMemory() - } - } - }() operationCount := 0 encodedBytes := 0 oldest := calls[0].enqueuedAt @@ -518,9 +503,6 @@ func completeCall[Request any, Response any]( err error, ) { call.resultOnce.Do(func() { - if !call.executing && call.releaseMemory != nil { - defer call.releaseMemory() - } result := batchResult[Response]{response: response, err: err} call.result <- result call.finishRecords(call.records) diff --git a/internal/service/batcher_completion_test.go b/internal/service/batcher_completion_test.go index b4b747d..390b378 100644 --- a/internal/service/batcher_completion_test.go +++ b/internal/service/batcher_completion_test.go @@ -4,6 +4,7 @@ import ( "github.com/liran/sink/internal/storage/memory" "context" + "slices" "strconv" "sync" "testing" @@ -30,6 +31,7 @@ func TestBatcherOldBatchCannotReleaseNewOwnerOfCompletedDocument(t *testing.T) { return []recordIdentity{key} } execute := func(ctx context.Context, calls []*batchCall[int, int]) { + slices.SortFunc(calls, func(a, b *batchCall[int, int]) int { return a.request - b.request }) for _, call := range calls { switch call.request { case 2: diff --git a/internal/service/batching_regressions_test.go b/internal/service/batching_regressions_test.go index ac3c5d2..ef62958 100644 --- a/internal/service/batching_regressions_test.go +++ b/internal/service/batching_regressions_test.go @@ -54,9 +54,6 @@ func TestRegressionMicrobatchMergeBudgetIsolation(t *testing.T) { server := completionServer(t, backend) server.server.maxReadBytes = 256 - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 makeCall := func(key string) *batchCall[*sink.WriteRequest, *sink.WriteResponse] { op := completionMerge(key, 1) op.GetMerge().LuaProgram.Source = []byte(`return function(current, incoming) return incoming end`) @@ -89,9 +86,6 @@ func TestRegressionMicrobatchReadBudgetIsolation(t *testing.T) { server := completionServer(t, backend) server.server.maxReadBytes = 256 - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 var calls []*batchCall[*sink.ReadRequest, *sink.ReadResponse] for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -128,9 +122,6 @@ func TestMicrobatchReadKeepsOversizedCallerIsolatedForSharedKey(t *testing.T) { server := completionServer(t, backend) server.server.maxReadBytes = 256 - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 var operations []*sink.ReadOperation for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -162,16 +153,13 @@ func TestMicrobatchReadKeepsOversizedCallerIsolatedForSharedKey(t *testing.T) { } } -func TestMicrobatchMergeBudgetFailureDoesNotLeakIntoNextCaller(t *testing.T) { +func TestMicrobatchMergesDoNotApplyResponseLimitToIntermediateDocuments(t *testing.T) { for _, present := range []bool{false, true} { t.Run(fmt.Sprint(present), func(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 150 - server.server.maxSnapshotBytes = 150 - - server.server.maxOutputBytes = 150 if present { for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -187,13 +175,13 @@ func TestMicrobatchMergeBudgetFailureDoesNotLeakIntoNextCaller(t *testing.T) { second := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, completionMerge("b", 1)) calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{first, second} server.executeWrites(t.Context(), calls) - oversized := awaitCompletion(t, first.result) + firstResult := awaitCompletion(t, first.result) healthy := awaitCompletion(t, second.result) - if oversized.err != nil || healthy.err != nil { - t.Fatalf("write errors: %v, %v", oversized.err, healthy.err) + if firstResult.err != nil || healthy.err != nil { + t.Fatalf("write errors: %v, %v", firstResult.err, healthy.err) } - if oversized.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED || oversized.response.Results[1].GetFailure().GetCode() != sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED { - t.Fatal(oversized.response) + if firstResult.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED || firstResult.response.Results[1].Status != sink.WriteStatus_WRITE_STATUS_APPLIED { + t.Fatal(firstResult.response) } if healthy.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED { t.Fatal(healthy.response) @@ -208,8 +196,8 @@ func TestMicrobatchMergeBudgetFailureDoesNotLeakIntoNextCaller(t *testing.T) { if err != nil { t.Fatal(err) } - if string(stored.Results[0].Document.Payload) != `{"value":1}` { - t.Fatalf("failed caller changed the folded state: %s", stored.Results[0].Document.Payload) + if string(stored.Results[0].Document.Payload) != `{"value":101}` { + t.Fatalf("folded chain lost a caller: %s", stored.Results[0].Document.Payload) } }) } @@ -221,11 +209,6 @@ func TestMicrobatchBudgetsSplitWithinExecutionMemoryLimit(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 - - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 - server.server.maxInFlightBytes = 1000 + failureResponseBytes(1) switch method { case "Write": var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] @@ -262,9 +245,6 @@ func TestMicrobatchBudgetsSplitWithinExecutionMemoryLimit(t *testing.T) { } } } - if server.server.inFlightBytes != 0 || server.server.inFlightRequests != 0 { - t.Fatal("execution reservation leaked") - } }) } } @@ -279,8 +259,6 @@ func TestMutationDispatcherPreservesDependenciesAcrossBatches(t *testing.T) { method := test.method backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 16), blocked: "product", release: make(chan struct{})} core := completionServer(t, backend).server - core.maxInFlightRequests = 2 - core.maxInFlightRequests = 2 opts := BatchingOptions{MaxOperations: 1, MaxWait: time.Millisecond} server, err := NewBatchingServer(core, opts) if err != nil { @@ -357,9 +335,6 @@ func TestMutationDispatcherPreservesDependenciesAcrossBatches(t *testing.T) { // active refresh wait without retaining queue/admission reservations. cancel() server.Close() - if core.inFlightBytes != 0 || core.inFlightRequests != 0 { - t.Fatal("shutdown leaked execution capacity") - } }) } } @@ -385,16 +360,13 @@ func enqueueCompletionCall[Request interface{ SizeVT() int }, Response any](t *t return done } -func TestMicrobatchConditionalWritesKeepEachCallersInputAndOutputBudget(t *testing.T) { +func TestMicrobatchConditionalWritesShareWorkingMemory(t *testing.T) { for _, kind := range []string{"Merge", "Replace"} { t.Run(kind, func(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 256 - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) @@ -427,67 +399,10 @@ func TestMicrobatchConditionalWritesKeepEachCallersInputAndOutputBudget(t *testi } } -func TestCancelledRPCIsOmittedFromLaterMemoryLimitedSegment(t *testing.T) { - backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 8), blocked: "a", release: make(chan struct{})} - server := completionServer(t, backend) - server.server.maxReadBytes = 256 - - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 - server.server.maxInFlightBytes = 1000 + failureResponseBytes(1) - cancelled, cancel := context.WithCancel(t.Context()) - defer cancel() - defer func() { - select { - case <-backend.release: - default: - close(backend.release) - } - }() - first := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, completionMerge("a", 1)) - second := completionWriteCall(cancelled, sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, completionMerge("b", 1)) - third := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, completionMerge("c", 1)) - calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{first, second, third} - done := make(chan struct{}) - go func() { server.executeWrites(t.Context(), calls); close(done) }() - event := awaitCompletion(t, backend.events) - if len(event.keys) != 1 || event.keys[0] != "a" { - t.Fatal(event) - } - cancel() - close(backend.release) - awaitCompletion(t, done) - for _, call := range []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{first, third} { - result := awaitCompletion(t, call.result) - if result.err != nil || result.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED { - t.Fatalf("live segment: %+v", result) - } - } - result := awaitCompletion(t, second.result) - if result.err == nil { - t.Fatal("cancelled segment was executed") - } - event = awaitCompletion(t, backend.events) - if len(event.keys) != 1 || event.keys[0] != "c" { - t.Fatalf("cancelled write reached backend: %+v", event) - } - select { - case extra := <-backend.events: - t.Fatalf("unexpected write: %+v", extra) - default: - } -} - func TestHotRecordSharesPhysicalReservationAcrossRPCBudgets(t *testing.T) { backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 8)} server := completionServer(t, backend) server.server.maxReadBytes = 256 - - server.server.maxSnapshotBytes = 256 - - server.server.maxOutputBytes = 256 - server.server.maxInFlightBytes = 2048 + failureResponseBytes(4) var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] for range 4 { calls = append(calls, completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, completionMerge("hot", 1))) diff --git a/internal/service/batching_server.go b/internal/service/batching_server.go index 3ed60ca..a946954 100644 --- a/internal/service/batching_server.go +++ b/internal/service/batching_server.go @@ -9,7 +9,6 @@ import ( "time" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" sinkmetrics "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/protocol" "google.golang.org/grpc/codes" @@ -54,8 +53,7 @@ func NewBatchingServer(server *Server, opts BatchingOptions) (*BatchingServer, e batching := &BatchingServer{server: server} readOptions := requestBatcherOptions[*sink.ReadRequest, *sink.ReadResponse]{ - MaxConcurrent: server.maxInFlightRequests, - Unlimited: server.memory != nil, + Unlimited: true, Method: "Read", MaxWait: normalized.MaxWait, MaxOperations: normalized.MaxOperations, @@ -64,14 +62,12 @@ func NewBatchingServer(server *Server, opts BatchingOptions) (*BatchingServer, e MaxQueuedBytes: normalized.MaxQueuedBytes, Execute: batching.executeReads, Metrics: normalized.Metrics, - ExecutionTimeout: server.requestTimeout, } readOptions.Store = server.boundStore batching.reads = newRequestBatcher(readOptions) writeOptions := requestBatcherOptions[*sink.WriteRequest, *sink.WriteResponse]{ - MaxConcurrent: server.maxInFlightRequests, - Unlimited: server.memory != nil, + Unlimited: true, Records: func(request *sink.WriteRequest) []recordIdentity { return mutationRequestRecords(request) }, @@ -86,14 +82,12 @@ func NewBatchingServer(server *Server, opts BatchingOptions) (*BatchingServer, e MaxQueuedBytes: normalized.MaxQueuedBytes, Execute: batching.executeWrites, Metrics: normalized.Metrics, - ExecutionTimeout: server.requestTimeout, } writeOptions.Store = server.boundStore batching.writes = newRequestBatcher(writeOptions) deleteOptions := requestBatcherOptions[*sink.DeleteRequest, *sink.DeleteResponse]{ - MaxConcurrent: server.maxInFlightRequests, - Unlimited: server.memory != nil, + Unlimited: true, Records: func(request *sink.DeleteRequest) []recordIdentity { return mutationRequestRecords(request) }, @@ -108,7 +102,6 @@ func NewBatchingServer(server *Server, opts BatchingOptions) (*BatchingServer, e MaxQueuedBytes: normalized.MaxQueuedBytes, Execute: batching.executeDeletes, Metrics: normalized.Metrics, - ExecutionTimeout: server.requestTimeout, } deleteOptions.Store = server.boundStore batching.deletes = newRequestBatcher(deleteOptions) @@ -164,7 +157,7 @@ func (s *BatchingServer) Read(ctx context.Context, req *sink.ReadRequest) (*sink if err := protocol.CheckStore(req, s.server.boundStore); err != nil { return nil, err } - ctx, cancel := executionContext(ctx, s.server.requestTimeout) + ctx, cancel := context.WithCancel(ctx) defer cancel() if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "read request must contain operations") @@ -172,11 +165,6 @@ func (s *BatchingServer) Read(ctx context.Context, req *sink.ReadRequest) (*sink if err := s.server.validateOperationCount(len(req.GetOperations())); err != nil { return nil, err } - ctx, release, err := s.server.beginBatchMemory(ctx, req) - if err != nil { - return nil, err - } - defer release() return s.reads.Submit(ctx, req, len(req.GetOperations()), req.SizeVT()) } @@ -184,7 +172,7 @@ func (s *BatchingServer) Write(ctx context.Context, req *sink.WriteRequest) (*si if err := protocol.CheckStore(req, s.server.boundStore); err != nil { return nil, err } - ctx, cancel := executionContext(ctx, s.server.requestTimeout) + ctx, cancel := context.WithCancel(ctx) defer cancel() if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "write request must contain operations") @@ -203,11 +191,6 @@ func (s *BatchingServer) Write(ctx context.Context, req *sink.WriteRequest) (*si return nil, status.Errorf(codes.InvalidArgument, "write request Lua programs: %v", err) } normalized := normalizeSynchronousWriteRequest(req, programs) - ctx, release, err := s.server.beginBatchMemory(ctx, normalized) - if err != nil { - return nil, err - } - defer release() return s.writes.Submit(ctx, normalized, len(normalized.GetOperations()), normalized.SizeVT()) } @@ -257,7 +240,7 @@ func (s *BatchingServer) Delete(ctx context.Context, req *sink.DeleteRequest) (* if err := protocol.CheckStore(req, s.server.boundStore); err != nil { return nil, err } - ctx, cancel := executionContext(ctx, s.server.requestTimeout) + ctx, cancel := context.WithCancel(ctx) defer cancel() if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "delete request must contain operations") @@ -271,64 +254,27 @@ func (s *BatchingServer) Delete(ctx context.Context, req *sink.DeleteRequest) (* if req.GetCompletionMode() == sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED { return s.server.Delete(ctx, req) } - ctx, release, err := s.server.beginBatchMemory(ctx, req) - if err != nil { - return nil, err - } - defer release() return s.deletes.Submit(ctx, req, len(req.GetOperations()), req.SizeVT()) } -func (s *BatchingServer) executeReads( - ctx context.Context, - calls []*batchCall[*sink.ReadRequest, *sink.ReadResponse], -) { +func (s *BatchingServer) executeReads(ctx context.Context, calls []*batchCall[*sink.ReadRequest, *sink.ReadResponse]) { calls = liveMutationCalls(calls) - limit := len(calls) - for len(calls) > 0 { - count := 0 - bytes := 2 * s.server.maxReadBytes - if s.server.memory != nil { - bytes = 0 - } - byteLimit := s.server.maxInFlightBytes - if s.server.memory != nil { - byteLimit = int(s.server.memory.Limit()) - } - for count < min(limit, len(calls)) { - next := calls[count].request.SizeVT() + failureResponseBytes(len(calls[count].request.GetOperations())) - if count > 0 && next > byteLimit-bytes { - break - } - bytes += next - count++ - } - group := liveMutationCalls(calls[:count]) - tail := calls[count:] - if len(group) == 0 { - calls = tail - continue - } - operations := make([]*sink.ReadOperation, 0, totalReadOperations(group)) - budgets := &requestBudgets{} - for _, call := range group { - operations = append(operations, call.request.GetOperations()...) - budgets.addContext(call.ctx, len(call.request.GetOperations())) - } - request := &sink.ReadRequest{Operations: operations} - execution, cancel := batchExecutionContext(ctx, group, s.server.requestTimeout) - outcome, err := s.server.read(execution, request, budgets) - cancel() - deferred := splitReadResponse(group, outcome, err) - if outcome.response != nil { - clear(outcome.response.Results) - } - if len(deferred) > 0 { - // Retry whole RPCs in smaller groups. No partial response escapes, - // and a deferred caller retains its own budget and repeated-key view. - limit = max(1, len(group)-len(deferred)) - } - calls = append(deferred, tail...) + if len(calls) == 0 { + return + } + operations := make([]*sink.ReadOperation, 0, totalReadOperations(calls)) + budgets := &requestBudgets{} + for _, call := range calls { + operations = append(operations, call.request.GetOperations()...) + budgets.addContext(call.ctx, len(call.request.GetOperations())) + } + request := &sink.ReadRequest{Operations: operations} + execution, cancel := batchExecutionContext(ctx, calls, 0) + defer cancel() + response, err := s.server.read(execution, request, budgets) + splitReadResponse(calls, response, err) + if response != nil { + clear(response.Results) } } @@ -342,39 +288,31 @@ func totalReadOperations(calls []*batchCall[*sink.ReadRequest, *sink.ReadRespons func splitReadResponse( calls []*batchCall[*sink.ReadRequest, *sink.ReadResponse], - outcome readOutcome, + response *sink.ReadResponse, err error, -) []*batchCall[*sink.ReadRequest, *sink.ReadResponse] { - response := outcome.response - if err == nil && (response == nil || len(response.Results) != totalReadOperations(calls) || - (len(outcome.deferred) != 0 && len(outcome.deferred) != len(calls))) { +) { + if err == nil && (response == nil || len(response.Results) != totalReadOperations(calls)) { err = status.Error(codes.Internal, "batched read returned an invalid result count") } if err != nil { for _, call := range calls { completeCall(call, (*sink.ReadResponse)(nil), err) } - return nil + return } - var deferred []*batchCall[*sink.ReadRequest, *sink.ReadResponse] offset := 0 - for owner, call := range calls { + for _, call := range calls { count := len(call.request.GetOperations()) - if len(outcome.deferred) != 0 && outcome.deferred[owner] { - deferred = append(deferred, call) - } else { - // Each caller owns its result slice; a slow receiver must not keep - // other callers' payloads alive through a shared backing array. - results := append([]*sink.ReadResult(nil), response.Results[offset:offset+count]...) - for index, result := range results { - result.OperationIndex = uint32(index) - } - split := &sink.ReadResponse{Results: results} - completeCall(call, split, nil) + // Each caller owns its result slice; a slow receiver must not keep + // other callers' payloads alive through a shared backing array. + results := append([]*sink.ReadResult(nil), response.Results[offset:offset+count]...) + for index, result := range results { + result.OperationIndex = uint32(index) } + split := &sink.ReadResponse{Results: results} + completeCall(call, split, nil) offset += count } - return deferred } func (s *BatchingServer) executeWrites( @@ -382,19 +320,7 @@ func (s *BatchingServer) executeWrites( calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse], ) { for _, wave := range planMutationWaves[*sink.WriteOperation](calls) { - parallel := len(wave.applied) > 0 && len(wave.visible) > 0 && - s.server.maxInFlightRequests > 1 - if parallel { - applied := combinedWriteRequest(wave.applied) - visible := combinedWriteRequest(wave.visible) - appliedBytes := s.server.estimateWriteExecution(applied, len(wave.applied), returningBatchCallers(wave.applied)).bytes - visibleBytes := s.server.estimateWriteExecution(visible, len(wave.visible), returningBatchCallers(wave.visible)).bytes - byteLimit := s.server.maxInFlightBytes - if s.server.memory != nil { - byteLimit = int(s.server.memory.Limit()) - } - parallel = appliedBytes+visibleBytes <= byteLimit - } + parallel := len(wave.applied) > 0 && len(wave.visible) > 0 var executions sync.WaitGroup for _, group := range [][]*batchCall[*sink.WriteRequest, *sink.WriteResponse]{wave.applied, wave.visible} { if len(group) == 0 { @@ -410,60 +336,21 @@ func (s *BatchingServer) executeWrites( } } -func (s *BatchingServer) executeWriteBatch( - ctx context.Context, - calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse], -) { +func (s *BatchingServer) executeWriteBatch(ctx context.Context, calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse]) { calls = liveMutationCalls(calls) if len(calls) == 0 { return } - ctx, cancel := batchExecutionContext(ctx, calls, s.server.requestTimeout) - defer cancel() - for start := 0; start < len(calls); { - byteLimit := s.server.maxInFlightBytes - if s.server.memory != nil { - byteLimit = int(s.server.memory.Limit()) - } - end := len(calls) - combined := combinedWriteRequest(calls[start:end]) - if s.server.estimateWriteExecution(combined, end-start, returningBatchCallers(calls[start:end])).bytes > byteLimit { - end = start + 1 - } - for end < len(calls) { - next := combinedWriteRequest(calls[start : end+1]) - if s.server.estimateWriteExecution(next, end+1-start, returningBatchCallers(calls[start:end+1])).bytes > byteLimit { - break - } - end++ - } - group := liveMutationCalls(calls[start:end]) - if len(group) == 0 { - start = end - continue - } - request := combinedWriteRequest(group) - budgets := &requestBudgets{} - for _, call := range group { - budgets.addContext(call.ctx, len(call.request.GetOperations())) - } - execution, executionCancel := batchExecutionContext(ctx, group, s.server.requestTimeout) - completion := newWriteCompletion(group, s.server.maxReadBytes) - response, err := s.server.write(execution, request, budgets, completion) - executionCancel() - completion.finish(response, err) - start = end - } -} - -func returningBatchCallers(calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse]) int { - count := 0 + request := combinedWriteRequest(calls) + budgets := &requestBudgets{} for _, call := range calls { - if hasWriteReturns(call.request) { - count++ - } + budgets.addContext(call.ctx, len(call.request.GetOperations())) } - return count + execution, cancel := batchExecutionContext(ctx, calls, 0) + defer cancel() + completion := newWriteCompletion(calls, s.server.maxReadBytes) + response, err := s.server.write(execution, request, budgets, completion) + completion.finish(response, err) } func combinedWriteRequest(calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse]) *sink.WriteRequest { @@ -491,18 +378,7 @@ func (s *BatchingServer) executeDeletes( calls []*batchCall[*sink.DeleteRequest, *sink.DeleteResponse], ) { for _, wave := range planMutationWaves[*sink.DeleteOperation](calls) { - parallel := len(wave.applied) > 0 && len(wave.visible) > 0 && - s.server.maxInFlightRequests > 1 - if parallel { - applied := combinedDeleteRequest(wave.applied) - visible := combinedDeleteRequest(wave.visible) - responseBytes := failureResponseBytes(len(applied.Operations) + len(visible.Operations)) - byteLimit := s.server.maxInFlightBytes - if s.server.memory != nil { - byteLimit = int(s.server.memory.Limit()) - } - parallel = applied.SizeVT()+visible.SizeVT()+responseBytes <= byteLimit - } + parallel := len(wave.applied) > 0 && len(wave.visible) > 0 var executions sync.WaitGroup for _, group := range [][]*batchCall[*sink.DeleteRequest, *sink.DeleteResponse]{wave.applied, wave.visible} { if len(group) == 0 { @@ -518,45 +394,20 @@ func (s *BatchingServer) executeDeletes( } } -func (s *BatchingServer) executeDeleteBatch( - ctx context.Context, - calls []*batchCall[*sink.DeleteRequest, *sink.DeleteResponse], -) { +func (s *BatchingServer) executeDeleteBatch(ctx context.Context, calls []*batchCall[*sink.DeleteRequest, *sink.DeleteResponse]) { calls = liveMutationCalls(calls) if len(calls) == 0 { return } - ctx, cancel := batchExecutionContext(ctx, calls, s.server.requestTimeout) - defer cancel() - for len(calls) > 0 { - count, bytes := 0, 0 - byteLimit := s.server.maxInFlightBytes - if s.server.memory != nil { - byteLimit = int(s.server.memory.Limit()) - } - for count < len(calls) { - next := calls[count].request.SizeVT() + failureResponseBytes(len(calls[count].request.GetOperations())) - if count > 0 && next > byteLimit-bytes { - break - } - bytes += next - count++ - } - group := liveMutationCalls(calls[:count]) - calls = calls[count:] - if len(group) == 0 { - continue - } - request := combinedDeleteRequest(group) - budgets := &requestBudgets{} - for _, call := range group { - budgets.addContext(call.ctx, len(call.request.GetOperations())) - } - execution, executionCancel := batchExecutionContext(ctx, group, s.server.requestTimeout) - response, err := s.server.delete(execution, request, budgets) - executionCancel() - splitDeleteResponse(group, response, err) + request := combinedDeleteRequest(calls) + budgets := &requestBudgets{} + for _, call := range calls { + budgets.addContext(call.ctx, len(call.request.GetOperations())) } + execution, cancel := batchExecutionContext(ctx, calls, 0) + defer cancel() + response, err := s.server.delete(execution, request, budgets) + splitDeleteResponse(calls, response, err) } func combinedDeleteRequest(calls []*batchCall[*sink.DeleteRequest, *sink.DeleteResponse]) *sink.DeleteRequest { @@ -609,21 +460,3 @@ func splitDeleteResponse( offset += count } } - -// In-process callers have no gRPC stats scope. Give them the same input ownership -// before queuing; Submit keeps a producer reference if the caller cancels. -func (s *Server) beginBatchMemory(ctx context.Context, req interface{ SizeVT() int }) (context.Context, func(), error) { - if s.memory == nil || capacity.FromContext(ctx) != nil { - return ctx, func() {}, nil - } - scope := s.memory.NewScope() - if err := scope.Admit(ctx, 2*req.SizeVT()+1024); err != nil { - scope.Release() - return ctx, nil, err - } - if err := scope.AdmitCompletion(ctx, protocol.CompletionBytes(req)); err != nil { - scope.Release() - return ctx, nil, err - } - return capacity.WithScope(ctx, scope), scope.Release, nil -} diff --git a/internal/service/batching_server_internal_test.go b/internal/service/batching_server_internal_test.go index c4f260b..585a3bf 100644 --- a/internal/service/batching_server_internal_test.go +++ b/internal/service/batching_server_internal_test.go @@ -17,8 +17,7 @@ func TestSplitReadResponseRejectsInvalidResultCount(t *testing.T) { } calls := []*batchCall[*sink.ReadRequest, *sink.ReadResponse]{call} response := &sink.ReadResponse{} - outcome := readOutcome{response: response} - splitReadResponse(calls, outcome, nil) + splitReadResponse(calls, response, nil) result := <-call.result if result.response != nil || status.Code(result.err) != codes.Internal { t.Fatalf("split result = %+v", result) diff --git a/internal/service/count_capacity_test.go b/internal/service/count_capacity_test.go deleted file mode 100644 index 7011083..0000000 --- a/internal/service/count_capacity_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package service_test - -import ( - "context" - "testing" - "time" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/merge" - "github.com/liran/sink/internal/service" - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" - "go.mongodb.org/mongo-driver/v2/bson" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type countCapacityStorage struct { - nativeFixtureStorage - entered chan storage.NativeRequest - release chan struct{} -} - -func (s *countCapacityStorage) Count(ctx context.Context, req storage.CountRequest) (storage.CountResponse, error) { - s.entered <- req.Request - select { - case <-s.release: - case <-ctx.Done(): - var empty storage.CountResponse - return empty, ctx.Err() - } - result := storage.CountResponse{Count: 123} - return result, nil -} - -func TestCountReservesBoundedResponsesAndReleasesCapacity(t *testing.T) { - for _, contentType := range []string{"application/json", "application/bson"} { - t.Run(contentType, func(t *testing.T) { - fixture := nativeFixtureStorage{Store: memory.New()} - backend := &countCapacityStorage{nativeFixtureStorage: fixture, entered: make(chan storage.NativeRequest, 33), release: make(chan struct{})} - luaOpts := merge.LuaOptions{} - engine, err := merge.NewLuaEngine(luaOpts) - if err != nil { - t.Fatal(err) - } - opts := service.Options{BoundStore: "primary", Storage: backend, Lua: engine, AdmissionWait: 10 * time.Millisecond, MaxInFlightRequests: 32} - server, err := service.New(opts) - if err != nil { - t.Fatal(err) - } - command := &sink.Command{Uri: "sink://primary", Method: "POST", Path: "/products/_search", ContentType: contentType} - want := 32 // Search counts can use every configured process request slot. - if contentType == "application/bson" { - fields := bson.D{{Key: "find", Value: "products"}} - payload, err := bson.Marshal(fields) - if err != nil { - t.Fatal(err) - } - command.Method, command.Path = "", "" - command.Uri, command.Payload = "sink://primary/catalog", payload - want = 5 // MongoDB still reserves its 48 MiB driver wire ceiling. - } - request := &sink.CountRequest{Command: command} - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - finished := make(chan error, want) - for range want { - go func() { _, err := server.Count(ctx, request); finished <- err }() - select { - case admitted := <-backend.entered: - if admitted.MaxBytes != 256<<10 { - t.Fatalf("backend did not receive the Count buffer limit: %d", admitted.MaxBytes) - } - case err := <-finished: - t.Fatalf("Count unnecessarily exhausted capacity: %v", err) - case <-ctx.Done(): - t.Fatal(ctx.Err()) - } - } - if _, err := server.Count(ctx, request); status.Code(err) != codes.ResourceExhausted { - t.Fatalf("Count bypassed concurrency or driver buffer limits: %v", err) - } - close(backend.release) - for range want { - if err := <-finished; err != nil { - t.Fatal(err) - } - } - response, err := server.Count(ctx, request) - if err != nil || response.GetCount() != 123 { - t.Fatalf("Count capacity was not released: %v, %v", response, err) - } - }) - } -} diff --git a/internal/service/failure_budget.go b/internal/service/failure_budget.go index 771b51a..5e00932 100644 --- a/internal/service/failure_budget.go +++ b/internal/service/failure_budget.go @@ -9,12 +9,6 @@ import ( const maxFailureMessageBytes = 1024 -// Reserve error text as well as result envelopes before parsing or executing. -// This also covers failures derived from stored documents, not just RPC input. -func failureResponseBytes(operations int) int { - return operations * (maxFailureMessageBytes + 128) -} - func failureMessageLimit(operations, maximum int) int { // Even tiny document quotas must retain a nonempty protocol diagnostic. return min(maxFailureMessageBytes, max(1, maximum/max(1, operations)-128)) diff --git a/internal/service/failure_budget_internal_test.go b/internal/service/failure_budget_internal_test.go index 4025ed1..a690933 100644 --- a/internal/service/failure_budget_internal_test.go +++ b/internal/service/failure_budget_internal_test.go @@ -10,8 +10,6 @@ import ( sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/storage" "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) type failureBudgetStorage struct { @@ -73,27 +71,6 @@ func TestCoalescedFailureBudgetsBelongToOriginalRPC(t *testing.T) { } } -func TestFailureSpaceIsReservedBeforeMutation(t *testing.T) { - backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 8)} - server := completionServer(t, backend).server - operation := completionPut("key", 1) - request := &sink.WriteRequest{Operations: []*sink.WriteOperation{operation}, CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} - server.maxInFlightBytes = request.SizeVT() + 100 - _, err := server.Write(t.Context(), request) - if status.Code(err) != codes.ResourceExhausted { - t.Fatalf("write ignored failure reservation: %v", err) - } - deleteOperation := &sink.DeleteOperation{Address: completionAddress("key")} - deleteRequest := &sink.DeleteRequest{Operations: []*sink.DeleteOperation{deleteOperation}, CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} - _, err = server.Delete(t.Context(), deleteRequest) - if status.Code(err) != codes.ResourceExhausted { - t.Fatalf("delete ignored failure reservation: %v", err) - } - if len(backend.events) != 0 { - t.Fatal("mutation reached storage without response reservation") - } -} - func TestTinyFailureBudgetsKeepNonemptyMessages(t *testing.T) { backend := failureBudgetStorage{Storage: memory.New()} server := completionServer(t, backend) diff --git a/internal/service/lua_compile_cancellation_test.go b/internal/service/lua_compile_cancellation_test.go index a7a8b1c..f96d8d0 100644 --- a/internal/service/lua_compile_cancellation_test.go +++ b/internal/service/lua_compile_cancellation_test.go @@ -31,15 +31,16 @@ func TestWriteCancellationInterruptsLuaCompilation(t *testing.T) { t.Fatal(err) } publisher := &recordingPublisher{} - options := service.Options{BoundStore: "primary", Storage: memory.New(), Lua: engine, Publisher: publisher, RequestTimeout: 20 * time.Millisecond} - if canceled { - options.RequestTimeout = 5 * time.Second - } + options := service.Options{BoundStore: "primary", Storage: memory.New(), Lua: engine, Publisher: publisher} server, err := service.New(options) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(t.Context()) + if !canceled { + cancel() + ctx, cancel = context.WithTimeout(t.Context(), 20*time.Millisecond) + } defer cancel() wantCode := codes.DeadlineExceeded if canceled { @@ -71,7 +72,7 @@ func TestLuaCompileLimitDoesNotCancelSiblingWrites(t *testing.T) { if err != nil { t.Fatal(err) } - options := service.Options{BoundStore: "primary", Storage: memory.New(), Lua: engine, RequestTimeout: 5 * time.Second} + options := service.Options{BoundStore: "primary", Storage: memory.New(), Lua: engine} server, err := service.New(options) if err != nil { t.Fatal(err) diff --git a/internal/service/memory_test.go b/internal/service/memory_test.go deleted file mode 100644 index 71e40ac..0000000 --- a/internal/service/memory_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package service - -import ( - "context" - "strings" - "testing" - "time" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" -) - -func TestBatchedInputMemoryIsNotReservedTwice(t *testing.T) { - opts := capacity.Options{Bytes: 512 << 10, BurstPercent: 10, WaitTimeout: time.Second} - pool, err := capacity.New(opts) - if err != nil { - t.Fatal(err) - } - scope := pool.NewScope() - defer scope.Release() - if err := scope.Admit(t.Context(), 256<<10); err != nil { - t.Fatal(err) - } - caller := capacity.WithScope(t.Context(), scope) - budgets := contextBudgets(caller, 1) - legacy := &admissionPool{requestTimeout: time.Second} - core := &Server{memory: pool, admissionPool: legacy} - request := admissionRequest{inputBytes: 256 << 10, resultBytes: 4096, sharedInput: budgets.ownsInput()} - _, release, err := core.admitRequest(t.Context(), request) - if err != nil { - t.Fatalf("batch reserved its already-owned input again: %v", err) - } - if pool.Used() >= 300<<10 { - t.Fatalf("batch charged payload copies instead of bookkeeping: %d", pool.Used()) - } - release() - if pool.Used() != 256<<10 { - t.Fatalf("batch released original caller ownership: %d", pool.Used()) - } -} - -func TestBatchCancellationRetainsProducerMemory(t *testing.T) { - opts := capacity.Options{Bytes: 1 << 20, BurstPercent: 10, WaitTimeout: time.Second} - pool, err := capacity.New(opts) - if err != nil { - t.Fatal(err) - } - scope := pool.NewScope() - if err := scope.Admit(t.Context(), 4096); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithCancel(capacity.WithScope(t.Context(), scope)) - defer cancel() - entered, release := make(chan struct{}), make(chan struct{}) - execute := func(_ context.Context, calls []*batchCall[int, int]) { - close(entered) - <-release - for _, call := range calls { - completeCall(call, 1, nil) - } - } - batching := requestBatcherOptions[int, int]{Unlimited: true, MaxWait: time.Millisecond, MaxOperations: 1, MaxBytes: 100, MaxQueuedOperations: 10, MaxQueuedBytes: 100, Execute: execute} - batcher := newRequestBatcher(batching) - done := make(chan error, 1) - go func() { _, err := batcher.Submit(ctx, 1, 1, 1); done <- err }() - <-entered - cancel() - <-done - scope.Release() - if pool.Used() != 4096 { - t.Fatal("canceled caller prematurely released producer input") - } - close(release) - batcher.Close() - if pool.Used() != 0 { - t.Fatal("producer memory leaked after completion") - } -} - -type demandReadStorage struct { - storage.Storage - entered chan struct{} - resume chan struct{} -} - -func (s *demandReadStorage) Read(ctx context.Context, req storage.ReadRequest) (storage.ReadResponse, error) { - s.entered <- struct{}{} - select { - case <-s.resume: - case <-ctx.Done(): - var empty storage.ReadResponse - return empty, ctx.Err() - } - return s.Storage.Read(ctx, req) -} - -func TestDemandReadsIgnoreMaximumResponseAndRequestCount(t *testing.T) { - backend := &demandReadStorage{Storage: memory.New(), entered: make(chan struct{}, 16), resume: make(chan struct{})} - core := completionServer(t, backend).server - options := capacity.Options{Bytes: 4 << 20, BurstPercent: 10, WaitTimeout: time.Second} - pool, err := capacity.New(options) - if err != nil { - t.Fatal(err) - } - core.memory = pool - core.maxReadBytes = 32 << 20 - core.maxInFlightRequests = 1 - done := make(chan error, 16) - for range 16 { - go func() { - operation := &sink.ReadOperation{Address: completionMerge("small", 1).Address} - request := &sink.ReadRequest{Operations: []*sink.ReadOperation{operation}} - _, err := core.Read(t.Context(), request) - done <- err - }() - } - for range 16 { - select { - case <-backend.entered: - case <-time.After(time.Second): - close(backend.resume) - t.Fatal("maximum response or count limit blocked small reads") - } - } - close(backend.resume) - for range 16 { - if err := <-done; err != nil { - t.Fatal(err) - } - } - if pool.Used() != 0 { - t.Fatalf("read leaked %d bytes", pool.Used()) - } -} - -func TestDemandReturnedWriteRejectsBeforeCommit(t *testing.T) { - backend := memory.New() - core := completionServer(t, backend).server - options := capacity.Options{Bytes: 128 << 10, BurstPercent: 10, WaitTimeout: time.Second} - pool, err := capacity.New(options) - if err != nil { - t.Fatal(err) - } - core.memory = pool - payload := []byte(`{"value":"` + strings.Repeat("x", 40<<10) + `"}`) - document := &sink.Document{Encoding: sink.DocumentEncoding_DOCUMENT_ENCODING_JSON, Payload: payload} - put := &sink.PutOperation{Document: document, Mode: sink.WriteMode_WRITE_MODE_UPSERT} - action := &sink.WriteOperation_Put{Put: put} - address := completionMerge("no-commit", 1).Address - operation := &sink.WriteOperation{Address: address, Action: action, ReturnDocument: true} - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, Operations: []*sink.WriteOperation{operation}} - result, err := core.Write(t.Context(), request) - if err != nil { - t.Fatal(err) - } - if result.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_FAILED || result.GetResults()[0].GetFailure().GetCode() != sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED { - t.Fatalf("unexpected failure: %v", result) - } - readOperation := &sink.ReadOperation{Address: address} - readRequest := &sink.ReadRequest{Operations: []*sink.ReadOperation{readOperation}} - read, err := core.Read(t.Context(), readRequest) - if err != nil || read.GetResults()[0].GetStatus() != sink.ReadStatus_READ_STATUS_NOT_FOUND { - t.Fatalf("write committed before reserving its return: %v %v", read, err) - } - if pool.Used() != 0 { - t.Fatal("failed write leaked") - } -} diff --git a/internal/service/merge_group_test.go b/internal/service/merge_group_test.go index d8b5957..c7a1e6d 100644 --- a/internal/service/merge_group_test.go +++ b/internal/service/merge_group_test.go @@ -140,11 +140,11 @@ func TestMergeFoldingBoundsConflictAttemptsAndOutput(t *testing.T) { want := sink.FailureCode_FAILURE_CODE_CONFLICT wantWrites := int64(2) if scenario == "output" { - want = sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED - wantWrites = 0 + want = sink.FailureCode_FAILURE_CODE_UNSPECIFIED + wantWrites = 1 } for _, result := range response.Results { - if result.GetFailure().GetCode() != want || result.Status == sink.WriteStatus_WRITE_STATUS_APPLIED { + if result.GetFailure().GetCode() != want || (scenario == "conflict" && result.Status == sink.WriteStatus_WRITE_STATUS_APPLIED) || (scenario == "output" && result.Status != sink.WriteStatus_WRITE_STATUS_APPLIED) { t.Fatalf("bounded result = %v, want %v", result, want) } } diff --git a/internal/service/mutation_batches_test.go b/internal/service/mutation_batches_test.go index 47d8b5e..99db148 100644 --- a/internal/service/mutation_batches_test.go +++ b/internal/service/mutation_batches_test.go @@ -78,7 +78,7 @@ func completionServer(t *testing.T, backend storage.Storage) *BatchingServer { if err != nil { t.Fatal(err) } - coreOpts := Options{BoundStore: "primary", Storage: backend, Lua: lua, RequestTimeout: 5 * time.Second} + coreOpts := Options{BoundStore: "primary", Storage: backend, Lua: lua} core, err := New(coreOpts) if err != nil { t.Fatal(err) @@ -341,34 +341,6 @@ func TestCompletionWavesKeepFullRecordAddressesSeparate(t *testing.T) { } } -func TestMixedCompletionFitsSingleRequestAndByteBudgets(t *testing.T) { - for _, limit := range []string{"global_requests", "store_requests", "bytes"} { - t.Run(limit, func(t *testing.T) { - backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 10)} - server := completionServer(t, backend) - switch limit { - case "global_requests": - server.server.maxInFlightRequests = 1 - case "store_requests": - server.server.maxInFlightRequests = 1 - case "bytes": - server.server.maxInFlightBytes = 3 * server.server.maxReadBytes - } - visible := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, completionMerge("visible", 1)) - applied := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, completionMerge("applied", 1)) - calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{visible, applied} - server.executeWrites(t.Context(), calls) - assertCompletionWrites(t, calls) - if first := awaitCompletion(t, backend.events); first.visible { - t.Fatal("limited execution did not run the applied group first") - } - if server.server.inFlightRequests != 0 || server.server.inFlightBytes != 0 { - t.Fatal("group admission leaked capacity") - } - }) - } -} - func TestDeleteCompletionChangesPreserveRecordOrder(t *testing.T) { backend := &completionStorage{Storage: memory.New(), events: make(chan completionEvent, 10)} server := completionServer(t, backend) diff --git a/internal/service/native.go b/internal/service/native.go index 7b57b7a..80290d3 100644 --- a/internal/service/native.go +++ b/internal/service/native.go @@ -107,13 +107,6 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - admission := admissionRequest{encodedBytes: nativeExecutionBytes(req.GetCommand(), request), stores: []string{protocol.CommandStore(req.GetCommand())}} - admission.inputBytes = req.SizeVT() - ctx, release, err := s.admitRequest(ctx, admission) - if err != nil { - return nil, err - } - defer release() result, err := backend.Execute(ctx, request) if err != nil { return nil, nativeStatus(err) @@ -154,9 +147,6 @@ func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanRes if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } - // Preserve the caller deadline across admission and backend execution. - ctx, cancel := executionContext(ctx, s.requestTimeout) - defer cancel() maximum = min(maximum, 4<<20) request, err := nativeRequest(req.GetCommand(), maximum) if err != nil { @@ -180,17 +170,6 @@ func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanRes if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - encodedBytes := nativeExecutionBytes(req.GetCommand(), request) + req.SizeVT() - req.GetCommand().SizeVT() - mediaType, _, _ := mime.ParseMediaType(request.ContentType) - if mediaType != "application/bson" { - encodedBytes += 2 * (storage.ScanBackendBytes(maximum) - maximum) - } - admission := admissionRequest{encodedBytes: encodedBytes, inputBytes: req.SizeVT(), stores: []string{protocol.CommandStore(req.GetCommand())}, scan: true, wait: true} - ctx, release, err := s.admitRequest(ctx, admission) - if err != nil { - return nil, err - } - defer release() if _, err := scan.Resume(); err != nil { return nil, nativeStatus(err) } @@ -217,17 +196,6 @@ func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanRes return response, nil } -func nativeExecutionBytes(req *sink.Command, request storage.NativeRequest) int { - bytes := req.SizeVT() + 16 + 2*request.MaxBytes - mediaType, _, _ := mime.ParseMediaType(request.ContentType) - if mediaType == "application/bson" { - // The driver receives a complete wire message before Sink can enforce - // its smaller document/page limit. Account for its 48 MiB wire ceiling. - bytes += 48 << 20 - } - return bytes -} - func (s *BatchingServer) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.ExecuteResponse, error) { return s.server.Execute(ctx, req) } diff --git a/internal/service/native_boundary_test.go b/internal/service/native_boundary_test.go new file mode 100644 index 0000000..177527a --- /dev/null +++ b/internal/service/native_boundary_test.go @@ -0,0 +1,162 @@ +package service + +import ( + "context" + "net/http" + "strings" + "testing" + + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/storage" + "github.com/liran/sink/internal/storage/memory" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type nativeBoundaryStore struct { + *memory.Store + calls int + request storage.NativeRequest + query storage.QueryRequest + scan storage.ScanRequest + failure error + output storage.NativeResponse + documents []storage.Document + more bool +} + +func (s *nativeBoundaryStore) Execute(_ context.Context, req storage.NativeRequest) (storage.NativeResponse, error) { + s.calls++ + s.request = req + return s.output, s.failure +} +func (s *nativeBoundaryStore) Query(_ context.Context, req storage.QueryRequest) (storage.QueryResponse, error) { + s.calls++ + s.query = req + s.request = req.Request + response := storage.QueryResponse{Documents: s.documents, HasMore: s.more} + return response, s.failure +} +func (s *nativeBoundaryStore) Count(_ context.Context, req storage.CountRequest) (storage.CountResponse, error) { + s.calls++ + s.request = req.Request + response := storage.CountResponse{Count: 123, Estimated: true} + return response, s.failure +} +func (s *nativeBoundaryStore) Scan(_ context.Context, req storage.ScanRequest) (storage.ScanResponse, error) { + s.calls++ + s.scan = req + s.request = req.Request + response := storage.ScanResponse{Documents: s.documents} + return response, s.failure +} + +func TestNativeBoundariesRetainWireLimitsAndCallerContext(t *testing.T) { + backend := &nativeBoundaryStore{Store: memory.New()} + server := completionServer(t, backend) + command := &sink.Command{Uri: "sink://primary", Method: "POST", Path: "/records/_search", ContentType: "application/json", Payload: []byte(`{}`)} + header := &sink.Header{Name: "X-Request", Values: []string{"one", "two"}} + command.Headers = []*sink.Header{header} + payload := []byte(`{"value":1}`) + document := storage.Document{Encoding: storage.DocumentEncodingJSON, Payload: payload} + backend.documents = []storage.Document{document} + backend.output = storage.NativeResponse{ContentType: "application/json", Payload: payload, StatusCode: 400, Headers: http.Header{"Warning": {"first", "second"}}} + execute := &sink.ExecuteRequest{Command: command} + response, err := server.Execute(t.Context(), execute) + if err != nil || response.GetStatusCode() != 400 || response.GetSuccess() || string(response.GetPayload()) != string(payload) || len(response.GetHeaders()[0].GetValues()) != 2 { + t.Fatalf("native response changed: %v %v", response, err) + } + if backend.request.Headers.Get("X-Request") != "one" { + t.Fatal("request headers lost") + } + projection := &sink.Projection{Fields: []string{"value"}} + field := &sink.SortField{Field: "value", Descending: true} + query := &sink.QueryRequest{Command: command, Page: 2, PageSize: 1, Sort: []*sink.SortField{field}, Projection: projection} + page, err := server.Query(t.Context(), query) + if err != nil || len(page.GetDocuments()) != 1 || backend.query.Offset != 1 || !backend.query.Sort[0].Descending || backend.query.Projection.Fields[0] != "value" { + t.Fatalf("query settings or page lost: %v %v", page, err) + } + count := &sink.CountRequest{Command: command} + total, err := server.Count(t.Context(), count) + if err != nil || total.GetCount() != 123 || !total.GetEstimated() || backend.request.MaxBytes != 256<<10 { + t.Fatalf("count bound/result: %v %v", total, err) + } + scan := &sink.ScanRequest{Command: command, Projection: projection} + scanned, err := server.Scan(t.Context(), scan) + if err != nil || len(scanned.GetDocuments()) != 1 || backend.scan.BatchSize != 100 || backend.scan.Request.MaxBytes >= 4<<20 { + t.Fatalf("scan page/bound: %v %v", scanned, err) + } + for _, code := range []codes.Code{codes.Canceled, codes.DeadlineExceeded, codes.Unavailable} { + backend.failure = status.Error(code, "backend stopped") + calls := []func() error{ + func() error { _, err := server.Execute(t.Context(), execute); return err }, + func() error { _, err := server.Query(t.Context(), query); return err }, + func() error { _, err := server.Count(t.Context(), count); return err }, + func() error { _, err := server.Scan(t.Context(), scan); return err }, + } + for _, call := range calls { + if err := call(); status.Code(err) != code { + t.Fatalf("native error changed: %v", err) + } + } + } + backend.failure = nil + server.server.maxReadBytes = 1024 + backend.output.Payload = []byte(strings.Repeat("x", 2048)) + if _, err := server.Execute(t.Context(), execute); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("oversized execute: %v", err) + } + backend.documents[0].Payload = backend.output.Payload + if _, err := server.Query(t.Context(), query); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("oversized query: %v", err) + } + if _, err := server.Scan(t.Context(), scan); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("oversized scan: %v", err) + } +} + +func TestNativeValidationStopsBeforeBackend(t *testing.T) { + backend := &nativeBoundaryStore{Store: memory.New()} + server := completionServer(t, backend) + invalid := []*sink.Command{ + {Uri: "invalid"}, {Uri: "sink://other"}, {Uri: "sink://primary", Method: string([]byte{0xff})}, + {Uri: "sink://primary", Payload: []byte(`{}`)}, {Uri: "sink://primary", ContentType: "application/json; =bad"}, + {Uri: "sink://primary", Headers: []*sink.Header{nil}}, + {Uri: "sink://primary", Headers: []*sink.Header{{Name: string([]byte{0xff}), Values: []string{"x"}}}}, + {Uri: "sink://primary", Headers: []*sink.Header{{Name: "X", Values: []string{string([]byte{0xff})}}}}, + } + for _, command := range invalid { + execute := &sink.ExecuteRequest{Command: command} + query := &sink.QueryRequest{Command: command} + count := &sink.CountRequest{Command: command} + scan := &sink.ScanRequest{Command: command} + calls := []func() error{ + func() error { _, err := server.Execute(t.Context(), execute); return err }, + func() error { _, err := server.Query(t.Context(), query); return err }, + func() error { _, err := server.Count(t.Context(), count); return err }, + func() error { _, err := server.Scan(t.Context(), scan); return err }, + } + for _, call := range calls { + if call() == nil { + t.Fatal("invalid command accepted") + } + } + } + command := &sink.Command{Uri: "sink://primary"} + query := &sink.QueryRequest{Command: command, PageSize: 1001} + if _, err := server.Query(t.Context(), query); status.Code(err) != codes.InvalidArgument { + t.Fatal(err) + } + scan := &sink.ScanRequest{Command: command, BatchSize: 1001} + if _, err := server.Scan(t.Context(), scan); status.Code(err) != codes.InvalidArgument { + t.Fatal(err) + } + scan.BatchSize = 1 + scan.Cursor = []byte("invalid") + if _, err := server.Scan(t.Context(), scan); status.Code(err) != codes.InvalidArgument { + t.Fatal(err) + } + if backend.calls != 0 { + t.Fatalf("invalid request reached backend: %d", backend.calls) + } +} diff --git a/internal/service/native_fixture_test.go b/internal/service/native_fixture_test.go deleted file mode 100644 index 773566b..0000000 --- a/internal/service/native_fixture_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package service_test - -import ( - "context" - "net/http" - - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" -) - -type nativeFixtureStorage struct { - *memory.Store - stopped chan struct{} - silent bool - started chan struct{} - queries chan storage.QueryRequest - scans chan storage.ScanRequest - counts chan storage.CountRequest - executeErr error - executeResponse *storage.NativeResponse -} - -func (s *nativeFixtureStorage) Query(_ context.Context, req storage.QueryRequest) (storage.QueryResponse, error) { - if s.queries != nil { - s.queries <- req - } - result := storage.QueryResponse{} - return result, nil -} - -func (s *nativeFixtureStorage) Count(_ context.Context, req storage.CountRequest) (storage.CountResponse, error) { - if s.counts != nil { - s.counts <- req - } - result := storage.CountResponse{Count: 123, Estimated: true} - return result, nil -} - -func (s *nativeFixtureStorage) Execute(_ context.Context, _ storage.NativeRequest) (storage.NativeResponse, error) { - if s.executeResponse != nil { - return *s.executeResponse, s.executeErr - } - response := storage.NativeResponse{ContentType: "application/json", Payload: []byte("{\"error\":\"native\"}\n"), StatusCode: 400, - Headers: http.Header{"Warning": {"first", "second"}}} - return response, s.executeErr -} - -func (s *nativeFixtureStorage) Scan(ctx context.Context, req storage.ScanRequest) (storage.ScanResponse, error) { - var response storage.ScanResponse - if s.scans != nil { - s.scans <- req - } - if s.silent { - close(s.started) - defer close(s.stopped) - <-ctx.Done() - return response, ctx.Err() - } - document := storage.Document{Encoding: storage.DocumentEncodingJSON, Payload: []byte(`{"value":1}`)} - response.Documents = []storage.Document{document} - return response, nil -} diff --git a/internal/service/operation_folding_test.go b/internal/service/operation_folding_test.go index 6ccf8a6..07023fa 100644 --- a/internal/service/operation_folding_test.go +++ b/internal/service/operation_folding_test.go @@ -321,11 +321,11 @@ func TestConditionalPutFoldingBoundsOutputAndConflicts(t *testing.T) { want := sink.FailureCode_FAILURE_CODE_CONFLICT writes := int64(2) if scenario == "output" { - want = sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED - writes = 0 + want = sink.FailureCode_FAILURE_CODE_UNSPECIFIED + writes = 1 } for _, result := range response.Results { - if result.GetFailure().GetCode() != want || result.Status == sink.WriteStatus_WRITE_STATUS_APPLIED { + if result.GetFailure().GetCode() != want || (scenario == "conflict" && result.Status == sink.WriteStatus_WRITE_STATUS_APPLIED) || (scenario == "output" && result.Status != sink.WriteStatus_WRITE_STATUS_APPLIED) { t.Fatalf("unbounded conditional put: %v", result) } } diff --git a/internal/service/options.go b/internal/service/options.go index 2e65f5c..1d098ab 100644 --- a/internal/service/options.go +++ b/internal/service/options.go @@ -3,9 +3,7 @@ package service import ( "errors" - "time" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/merge" sinkmetrics "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/queue" @@ -15,28 +13,14 @@ import ( const defaultMaxMergeAttempts = 3 type Options struct { - Memory *capacity.Pool - BoundStore string - Storage storage.Storage - Lua *merge.LuaEngine - Publisher queue.Publisher - MaxOperations int - MaxMergeAttempts int - Metrics *sinkmetrics.Metrics - RequestTimeout time.Duration - MaxInFlightRequests int - MaxInFlightBytes int - MaxAdmissionRequests int - MaxAdmissionBytes int - AdmissionWait time.Duration - MaxPublishRequests int - MaxPublishBytes int - MaxReadBytes int - MaxSnapshotBytes int - MaxOutputBytes int - MaxScanRequests int - MaxScanBytes int - ScanAdmissionWait time.Duration + BoundStore string + Storage storage.Storage + Lua *merge.LuaEngine + Publisher queue.Publisher + MaxOperations int + MaxMergeAttempts int + MaxReadBytes int + Metrics *sinkmetrics.Metrics } func New(opts Options) (*Server, error) { @@ -49,120 +33,18 @@ func New(opts Options) (*Server, error) { if opts.Lua == nil { return nil, errors.New("create Sink server: Lua merge engine is required") } - if opts.MaxOperations < 0 { - return nil, errors.New("create Sink server: max operations cannot be negative") + if opts.MaxOperations < 0 || opts.MaxMergeAttempts < 0 || opts.MaxReadBytes < 0 { + return nil, errors.New("create Sink server: limits cannot be negative") } - if opts.MaxMergeAttempts < 0 { - return nil, errors.New("create Sink server: max merge attempts cannot be negative") + if opts.MaxOperations == 0 { + opts.MaxOperations = int(^uint(0) >> 1) } - if opts.RequestTimeout < 0 || opts.MaxInFlightRequests < 0 || opts.MaxInFlightBytes < 0 || opts.MaxReadBytes < 0 { - return nil, errors.New("create Sink server: resource limits cannot be negative") - } - - if opts.MaxInFlightRequests == 0 { - opts.MaxInFlightRequests = 128 - } - if opts.MaxInFlightBytes == 0 { - opts.MaxInFlightBytes = 256 << 20 - } - if opts.MaxSnapshotBytes < 0 || opts.MaxOutputBytes < 0 { - return nil, errors.New("execution budgets cannot be negative") + if opts.MaxMergeAttempts == 0 { + opts.MaxMergeAttempts = defaultMaxMergeAttempts } if opts.MaxReadBytes == 0 { opts.MaxReadBytes = storage.DefaultMaxReadBytes } - if opts.MaxSnapshotBytes == 0 { - opts.MaxSnapshotBytes = opts.MaxReadBytes - } - if opts.MaxOutputBytes == 0 { - opts.MaxOutputBytes = opts.MaxReadBytes - } - if opts.MaxAdmissionRequests < 0 || opts.MaxAdmissionBytes < 0 || opts.AdmissionWait < 0 { - return nil, errors.New("create Sink server: admission queue limits cannot be negative") - } - if opts.MaxAdmissionRequests == 0 { - opts.MaxAdmissionRequests = 1024 - } - if opts.MaxAdmissionBytes == 0 { - opts.MaxAdmissionBytes = min(32<<20, opts.MaxInFlightBytes) - } - if opts.AdmissionWait == 0 { - opts.AdmissionWait = 2 * time.Second - } - if opts.RequestTimeout > 0 { - opts.AdmissionWait = min(opts.AdmissionWait, opts.RequestTimeout) - } - if opts.MaxPublishRequests < 0 || opts.MaxPublishBytes < 0 { - return nil, errors.New("create Sink server: publish limits cannot be negative") - } - if opts.MaxPublishRequests == 0 { - opts.MaxPublishRequests = 32 - } - if opts.MaxPublishBytes == 0 { - opts.MaxPublishBytes = 256 << 20 - } - if opts.MaxScanRequests < 0 || opts.MaxScanBytes < 0 || opts.ScanAdmissionWait < 0 { - return nil, errors.New("create Sink server: scan limits cannot be negative") - } - if opts.MaxScanRequests == 0 { - opts.MaxScanRequests = max(1, opts.MaxInFlightRequests/2) - } - if opts.MaxScanBytes == 0 { - opts.MaxScanBytes = max(1, opts.MaxInFlightBytes/2) - } - if opts.ScanAdmissionWait == 0 { - opts.ScanAdmissionWait = 2 * time.Second - } - if opts.RequestTimeout > 0 { - opts.ScanAdmissionWait = min(opts.ScanAdmissionWait, opts.RequestTimeout) - } - if opts.MaxScanRequests > opts.MaxInFlightRequests || opts.MaxScanBytes > opts.MaxInFlightBytes { - return nil, errors.New("create Sink server: scan limits cannot exceed total limits") - } - - maxOperations := opts.MaxOperations - if maxOperations == 0 { - maxOperations = int(^uint(0) >> 1) - } - maxMergeAttempts := opts.MaxMergeAttempts - if maxMergeAttempts == 0 { - maxMergeAttempts = defaultMaxMergeAttempts - } - - executionAdmission := &admissionPool{ - name: "execution", - metrics: opts.Metrics, - requestTimeout: opts.RequestTimeout, - maxInFlightRequests: opts.MaxInFlightRequests, - maxInFlightBytes: opts.MaxInFlightBytes, - maxScanRequests: opts.MaxScanRequests, - maxScanBytes: opts.MaxScanBytes, - scanAdmissionWait: opts.ScanAdmissionWait, - maxQueuedRequests: opts.MaxAdmissionRequests, - maxQueuedBytes: opts.MaxAdmissionBytes, - admissionWait: opts.AdmissionWait, - } - publishAdmission := &admissionPool{ - name: "publish", - metrics: opts.Metrics, - requestTimeout: opts.RequestTimeout, - maxInFlightRequests: opts.MaxPublishRequests, - maxInFlightBytes: opts.MaxPublishBytes, - } - server := &Server{ - boundStore: opts.BoundStore, - memory: opts.Memory, - storage: opts.Storage, - lua: opts.Lua, - publisher: opts.Publisher, - maxOperations: maxOperations, - maxMergeAttempts: maxMergeAttempts, - metrics: opts.Metrics, - maxReadBytes: opts.MaxReadBytes, - maxSnapshotBytes: opts.MaxSnapshotBytes, - maxOutputBytes: opts.MaxOutputBytes, - admissionPool: executionAdmission, - publishAdmission: publishAdmission, - } + server := &Server{boundStore: opts.BoundStore, storage: opts.Storage, lua: opts.Lua, publisher: opts.Publisher, maxOperations: opts.MaxOperations, maxMergeAttempts: opts.MaxMergeAttempts, metrics: opts.Metrics, maxReadBytes: opts.MaxReadBytes} return server, nil } diff --git a/internal/service/options_test.go b/internal/service/options_test.go deleted file mode 100644 index 3a0aec6..0000000 --- a/internal/service/options_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package service - -import ( - "testing" - - "github.com/liran/sink/internal/merge" - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestPublishAndExecutionEnforceIndependentStoreRequestLimits(t *testing.T) { - luaOptions := merge.LuaOptions{} - lua, err := merge.NewLuaEngine(luaOptions) - if err != nil { - t.Fatal(err) - } - for _, executionLimit := range []int{1, 2} { - publishLimit := 3 - executionLimit - opts := Options{BoundStore: "primary", Storage: memory.New(), Lua: lua, MaxInFlightRequests: executionLimit, MaxPublishRequests: publishLimit} - server, err := New(opts) - if err != nil { - t.Fatal(err) - } - for _, publish := range []bool{false, true} { - limit := executionLimit - if publish { - limit = publishLimit - } - request := admissionRequest{encodedBytes: 1, stores: []string{"primary"}, publish: publish} - for range limit { - _, release, err := server.admitRequest(t.Context(), request) - if err != nil { - t.Fatalf("publish=%v rejected before its own limit: %v", publish, err) - } - t.Cleanup(release) - } - _, release, err := server.admitRequest(t.Context(), request) - if release != nil { - release() - } - if status.Code(err) != codes.ResourceExhausted { - t.Fatalf("publish=%v exceeded its own store limit: %v", publish, err) - } - } - } -} diff --git a/internal/service/publish.go b/internal/service/publish.go index 0a94d3f..06bc46c 100644 --- a/internal/service/publish.go +++ b/internal/service/publish.go @@ -5,7 +5,6 @@ import ( "errors" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/queue" "github.com/liran/sink/internal/storage" "google.golang.org/grpc/codes" @@ -46,22 +45,6 @@ func (s *Server) publishWrites( mutation := queue.Mutation{Write: cloned} mutations = append(mutations, mutation) } - // Kafka serializes every mutation separately before enqueueing. Referenced - // Lua declarations therefore expand beyond the original RPC's wire size. - encodedBytes := 0 - for _, mutation := range mutations { - encodedBytes += queue.MutationSize(mutation) + len(mutation.Write.GetAddress().GetUri()) - } - lease := capacity.FromContext(ctx).NewLease() - defer capacity.Close(lease) - if lease != nil { - if err := lease.Grow(ctx, int64(encodedBytes), capacity.Request); err != nil { - for _, operation := range operations { - setWriteFailure(results[operation.index], sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED, err, true) - } - return nil - } - } request := queue.PublishRequest{Mutations: mutations} response, err := s.publisher.Publish(ctx, request) if err != nil { diff --git a/internal/service/publish_admission_test.go b/internal/service/publish_admission_test.go deleted file mode 100644 index d703000..0000000 --- a/internal/service/publish_admission_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package service - -import ( - "context" - "testing" - "testing/synctest" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/queue" - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type admissionPublisher struct { - entered chan struct{} - release chan struct{} - calls int -} - -func (p *admissionPublisher) Publish(ctx context.Context, req queue.PublishRequest) (queue.PublishResponse, error) { - response := queue.PublishResponse{Results: make([]queue.PublishResult, len(req.Mutations))} - p.calls++ - if p.entered != nil { - p.entered <- struct{}{} - select { - case <-p.release: - case <-ctx.Done(): - return response, ctx.Err() - } - } - for index := range response.Results { - response.Results[index].Status = queue.PublishStatusAccepted - } - return response, nil -} - -func TestPublishingContinuesBehindSynchronousByteWaiters(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := completionServer(t, memory.New()) - publisher := &admissionPublisher{} - server.server.publisher = publisher - server.server.maxInFlightBytes = 100 - initial := admissionRequest{encodedBytes: 80} - _, release, err := server.server.admitRequest(t.Context(), initial) - if err != nil { - t.Fatal(err) - } - defer release() - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - waiting := admissionRequest{encodedBytes: 90, wait: true} - done := make(chan error, 1) - go func() { - _, cleanup, err := server.server.admitRequest(ctx, waiting) - if cleanup != nil { - cleanup() - } - done <- err - }() - synctest.Wait() - for range 200 { - operation := completionMerge("async", 1) - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED, Operations: []*sink.WriteOperation{operation}} - response, err := server.Write(t.Context(), request) - if err != nil || response.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_ACCEPTED { - t.Fatalf("synchronous waiter blocked publishing: response=%v error=%v", response, err) - } - } - address := completionPut("async", 1).Address - operation := &sink.DeleteOperation{Address: address} - request := &sink.DeleteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED, Operations: []*sink.DeleteOperation{operation}} - response, err := server.Delete(t.Context(), request) - if err != nil || response.GetResults()[0].GetStatus() != sink.DeleteStatus_DELETE_STATUS_ACCEPTED { - t.Fatalf("synchronous waiter blocked delete publishing: response=%v error=%v", response, err) - } - cancel() - if status.Code(<-done) != codes.Canceled || publisher.calls != 201 { - t.Fatal("waiter cancellation or publish count changed") - } - if server.server.publishAdmission.inFlightBytes != 0 || server.server.publishAdmission.inFlightRequests != 0 { - t.Fatal("completed publishes leaked capacity") - } - }) -} - -func TestPublishSaturationPreservesDurabilityAndSynchronousCapacity(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := completionServer(t, memory.New()) - publisher := &admissionPublisher{entered: make(chan struct{}, 1), release: make(chan struct{})} - server.server.publisher = publisher - server.server.publishAdmission.maxInFlightRequests = 1 - operation := completionPut("async", 1) - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED, Operations: []*sink.WriteOperation{operation}} - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { - _, err := server.Write(ctx, request) - done <- err - }() - <-publisher.entered - synctest.Wait() - if len(done) != 0 { - t.Fatal("acknowledged before the publisher completed") - } - _, err := server.Write(t.Context(), request) - if status.Code(err) != codes.ResourceExhausted || publisher.calls != 1 { - t.Fatalf("publish limit did not reject before enqueue: %v", err) - } - synchronous := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, Operations: []*sink.WriteOperation{operation}} - response, err := server.Write(t.Context(), synchronous) - if err != nil || response.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_APPLIED { - t.Fatalf("stalled publisher blocked synchronous storage: response=%v error=%v", response, err) - } - cancel() - if <-done == nil { - t.Fatal("cancelled publish was acknowledged") - } - if server.server.publishAdmission.inFlightBytes != 0 || server.server.publishAdmission.inFlightRequests != 0 { - t.Fatal("cancelled publish leaked capacity") - } - close(publisher.release) - response, err = server.Write(t.Context(), request) - if err != nil || response.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_ACCEPTED { - t.Fatalf("publish capacity did not recover: response=%v error=%v", response, err) - } - }) -} - -func TestPublishByteBudgetRejectsBeforeEnqueue(t *testing.T) { - server := completionServer(t, memory.New()) - publisher := &admissionPublisher{} - server.server.publisher = publisher - operation := completionPut("async", 1) - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED, Operations: []*sink.WriteOperation{operation}} - server.server.publishAdmission.maxInFlightBytes = request.SizeVT() - 1 - _, err := server.Write(t.Context(), request) - if status.Code(err) != codes.ResourceExhausted || publisher.calls != 0 { - t.Fatalf("publish exceeded byte budget: calls=%d error=%v", publisher.calls, err) - } -} - -type admissionWriteStorage struct { - storage.Storage - batches int -} - -func (s *admissionWriteStorage) Write(ctx context.Context, req storage.WriteRequest) (storage.WriteResponse, error) { - s.batches++ - return s.Storage.Write(ctx, req) -} - -func TestMixedWriteBatchReservesOnlyReturningCallers(t *testing.T) { - backend := &admissionWriteStorage{Storage: memory.New()} - server := completionServer(t, backend) - server.server.maxReadBytes = 1024 - mode := sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED - calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{ - completionWriteCall(t.Context(), mode, completionPut("returned", 1)), - completionWriteCall(t.Context(), mode, completionPut("plain-a", 2)), - completionWriteCall(t.Context(), mode, completionPut("plain-b", 3)), - completionWriteCall(t.Context(), mode, completionPut("plain-c", 4)), - } - calls[0].request.Operations[0].ReturnDocument = true - combined := combinedWriteRequest(calls) - server.server.maxInFlightBytes = combined.SizeVT() + failureResponseBytes(len(combined.Operations)) + server.server.maxReadBytes - server.executeWriteBatch(t.Context(), calls) - for index, call := range calls { - response := requireWriteResult(t, call) - if (response.Results[0].GetDocument() != nil) != (index == 0) { - t.Fatalf("caller %d received the wrong returned-document behavior", index) - } - } - if backend.batches != 1 { - t.Fatalf("non-returning callers reserved response space and split the batch: batches=%d", backend.batches) - } -} diff --git a/internal/service/publish_memory_test.go b/internal/service/publish_memory_test.go deleted file mode 100644 index 4f2a376..0000000 --- a/internal/service/publish_memory_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package service - -import ( - "context" - "crypto/sha256" - "fmt" - "strings" - "testing" - "time" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" - "github.com/liran/sink/internal/protocol" - "github.com/liran/sink/internal/queue" - "github.com/liran/sink/internal/storage/memory" -) - -type expansionPublisher struct { - pool *capacity.Pool - sourceBytes int - used int64 - unique map[*byte]bool -} - -func (p *expansionPublisher) Publish(_ context.Context, req queue.PublishRequest) (queue.PublishResponse, error) { - p.used = p.pool.Used() - response := queue.PublishResponse{Results: make([]queue.PublishResult, len(req.Mutations))} - for i, mutation := range req.Mutations { - source := mutation.Write.GetMerge().GetLuaProgram().GetSource() - p.sourceBytes += len(source) - p.unique[&source[0]] = true - response.Results[i].Status = queue.PublishStatusAccepted - } - return response, nil -} - -func TestSharedLuaExpansionStaysInsideMemoryCapacity(t *testing.T) { - for _, maximum := range []int64{2 << 20, 32 << 20} { - t.Run(fmt.Sprint(maximum), func(t *testing.T) { - options := capacity.Options{Bytes: maximum, BurstPercent: 10, WaitTimeout: 20 * time.Millisecond} - pool, err := capacity.New(options) - if err != nil { - t.Fatal(err) - } - server := completionServer(t, memory.New()).server - server.memory = pool - publisher := &expansionPublisher{pool: pool, unique: make(map[*byte]bool)} - server.publisher = publisher - source := []byte("--" + strings.Repeat("x", 60000) + "\nreturn function(current, incoming) return incoming end") - digest := sha256.Sum256(source) - declaration := &sink.LuaProgram{Source: source, Sha256: digest[:]} - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED, LuaPrograms: []*sink.LuaProgram{declaration}} - for i := range 256 { - operation := completionMerge(fmt.Sprint("expanded-", i), 1) - reference := &sink.LuaProgram{Sha256: digest[:]} - operation.GetMerge().LuaProgram = reference - request.Operations = append(request.Operations, operation) - } - scope := pool.NewScope() - defer scope.Release() - ctx := capacity.WithScope(t.Context(), scope) - handler := func(ctx context.Context, req any) (any, error) { return server.Write(ctx, req.(*sink.WriteRequest)) } - response, err := protocol.MemoryInterceptor(ctx, request, nil, handler) - if err != nil { - t.Fatal(err) - } - result := response.(*protocol.ManagedMessage).Message.(*sink.WriteResponse) - for _, op := range result.Results { - if maximum == 2<<20 { - if op.GetStatus() != sink.WriteStatus_WRITE_STATUS_FAILED || op.GetFailure().GetCode() != sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED || !op.GetFailure().GetRetryable() { - t.Fatalf("expanded source was not rejected before enqueue: %v", op) - } - } else if op.GetStatus() != sink.WriteStatus_WRITE_STATUS_ACCEPTED { - t.Fatalf("sufficient capacity rejected the request: %v", op) - } - } - if maximum == 2<<20 && publisher.sourceBytes != 0 { - t.Fatal("expanded mutation reached the publisher before reserving capacity") - } - if maximum == 32<<20 { - if len(publisher.unique) != 1 { - t.Fatalf("shared declaration became %d independent copies", len(publisher.unique)) - } - if publisher.used < int64(publisher.sourceBytes) || publisher.used > maximum { - t.Fatalf("encoded payload not covered: used=%d payload=%d", publisher.used, publisher.sourceBytes) - } - } - scope.Release() - if pool.Used() != 0 { - t.Fatal("completed publish leaked memory capacity") - } - }) - } -} diff --git a/internal/service/query.go b/internal/service/query.go index 04a62e1..366fd3f 100644 --- a/internal/service/query.go +++ b/internal/service/query.go @@ -48,14 +48,6 @@ func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.Query if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - encodedBytes := nativeExecutionBytes(req.GetCommand(), request) + req.SizeVT() - req.GetCommand().SizeVT() - admission := admissionRequest{encodedBytes: encodedBytes, stores: []string{protocol.CommandStore(req.GetCommand())}} - admission.inputBytes = req.SizeVT() - ctx, release, err := s.admitRequest(ctx, admission) - if err != nil { - return nil, err - } - defer release() result, err := backend.Query(ctx, query) if err != nil { return nil, nativeStatus(err) @@ -83,7 +75,7 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count return nil, err } // Count retains only totals and bounded backend metadata. Enforce the - // smaller buffer in the adapter as well as reserving it at admission. + // smaller buffer in the adapter for this bounded scalar response. request, err := nativeRequest(req.GetCommand(), min(maximum, maxCountResponseBytes)) if err != nil { return nil, err @@ -92,13 +84,6 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count if !ok { return nil, nativeStatus(storage.ErrNativeUnsupported) } - admission := admissionRequest{encodedBytes: nativeExecutionBytes(req.GetCommand(), request), stores: []string{protocol.CommandStore(req.GetCommand())}} - admission.inputBytes = req.SizeVT() - ctx, release, err := s.admitRequest(ctx, admission) - if err != nil { - return nil, err - } - defer release() countRequest := storage.CountRequest{Request: request} count, err := backend.Count(ctx, countRequest) if err != nil { diff --git a/internal/service/read.go b/internal/service/read.go index c277d12..3b3c18b 100644 --- a/internal/service/read.go +++ b/internal/service/read.go @@ -2,10 +2,8 @@ package service import ( "context" - "errors" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" "github.com/liran/sink/internal/storage" @@ -17,52 +15,24 @@ func (s *Server) Read(ctx context.Context, req *sink.ReadRequest) (*sink.ReadRes if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } - outcome, err := s.read(ctx, req, contextBudgets(ctx, len(req.GetOperations()))) - if err != nil { - return nil, err - } - return outcome.response, nil + return s.read(ctx, req, contextBudgets(ctx, len(req.GetOperations()))) } -type readOutcome struct { - response *sink.ReadResponse - deferred []bool -} - -func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *requestBudgets) (readOutcome, error) { - var outcome readOutcome +func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *requestBudgets) (*sink.ReadResponse, error) { if req == nil || len(req.GetOperations()) == 0 { - return outcome, status.Error(codes.InvalidArgument, "read request must contain operations") + return nil, status.Error(codes.InvalidArgument, "read request must contain operations") } if err := s.validateOperationCount(len(req.GetOperations())); err != nil { - return outcome, err - } - admission := admissionRequest{encodedBytes: req.SizeVT() + failureResponseBytes(len(req.GetOperations())) + 2*s.maxReadBytes, stores: operationStores(req.GetOperations()), wait: budgets != nil} - admission.inputBytes = req.SizeVT() + 128*len(req.GetOperations()) - admission.resultBytes = failureResponseBytes(len(req.GetOperations())) - admission.sharedInput = budgets.ownsInput() - ctx, release, err := s.admitRequest(ctx, admission) - if err != nil { - return outcome, err - } - defer release() - if budgets == nil { - budgets = contextBudgets(ctx, len(req.GetOperations())) - } - if budgets != nil { - budgets.producer = capacity.FromContext(ctx) + return nil, err } response := &sink.ReadResponse{ Results: make([]*sink.ReadResult, len(req.GetOperations())), } defer boundResultFailures(response.Results, budgets, s.maxReadBytes) - outcome.response = response - outcome.deferred = make([]bool, budgets.callerCount()) storageOperations := make([]storage.ReadOperation, 0, len(req.GetOperations())) operationIndexes := make([]int, 0, len(req.GetOperations())) storageIndexes := make([]int, 0, len(req.GetOperations())) - owners := make([][]int, 0, len(req.GetOperations())) positions := make(map[recordIdentity]int) for index, operation := range req.GetOperations() { @@ -84,53 +54,30 @@ func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *reque positions[key] = position storageOperation := storage.ReadOperation{Address: address} storageOperations = append(storageOperations, storageOperation) - owners = append(owners, nil) } - owners[position] = append(owners[position], budgets.owner(index)) operationIndexes = append(operationIndexes, index) storageIndexes = append(storageIndexes, position) } if len(storageOperations) == 0 { - return outcome, nil + return response, nil } - snapshotBudgets := budgets.fresh(forwarding.Snapshots, s.maxReadBytes) - working := storage.NewReadBudget(s.maxReadBytes) - snapshots := capacity.FromContext(ctx).NewLease() - defer capacity.Close(snapshots) - for index := range storageOperations { - budget := sharedSnapshotBudget(owners[index], snapshotBudgets) - if budgets.callerCount() > 1 { - budget = storage.NewWorkingSetReadBudget(budget, working) - } - storageOperations[index].Budget = storage.WithMemoryBudget(ctx, budget, snapshots) - } - storageRequest := storage.ReadRequest{Operations: storageOperations, Budget: storage.NewReadBudget(s.maxReadBytes)} + budget := storage.NewUnboundedReadBudget() + storageRequest := storage.ReadRequest{Operations: storageOperations, Budget: budget} storageResponse, err := s.storage.Read(ctx, storageRequest) if err != nil { - return outcome, status.Errorf(codes.Unavailable, "read records: %v", err) + return nil, status.Errorf(codes.Unavailable, "read records: %v", err) } if len(storageResponse.Results) != len(storageOperations) { - return outcome, status.Error(codes.Internal, "storage returned an invalid read result count") + return nil, status.Error(codes.Internal, "storage returned an invalid read result count") } defer clear(storageResponse.Results) - for index, operationIndex := range operationIndexes { - stored := storageResponse.Results[storageIndexes[index]] - if errors.Is(stored.Err, storage.ErrReadWorkingSetFull) { - outcome.deferred[budgets.owner(operationIndex)] = true - } - } - // Charge copies in each original RPC's order, including repeated keys. // Check the aggregate output size before allocating any document copies. outputBudgets := budgets.fresh(forwarding.Returns, s.maxReadBytes) - outputBytes := make([]int, budgets.callerCount()) for index, operationIndex := range operationIndexes { owner := budgets.owner(operationIndex) - if outcome.deferred[owner] { - continue - } stored := storageResponse.Results[storageIndexes[index]] if stored.Status == storage.ReadStatusFound { if err := outputBudgets[owner].Reserve(len(stored.Document.Payload)); err != nil { @@ -138,30 +85,15 @@ func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *reque setReadFailure(response.Results[operationIndex], code, err, retryable) continue } - outputBytes[owner] += len(stored.Document.Payload) + 128 - } - } - remaining := s.maxReadBytes - for owner, size := range outputBytes { - if size > remaining { - outcome.deferred[owner] = true - } else { - remaining -= size } } for index, operationIndex := range operationIndexes { result := response.Results[operationIndex] - if outcome.deferred[budgets.owner(operationIndex)] || result.Status != sink.ReadStatus_READ_STATUS_UNSPECIFIED { + if result.Status != sink.ReadStatus_READ_STATUS_UNSPECIFIED { continue } stored := storageResponse.Results[storageIndexes[index]] - if stored.Status == storage.ReadStatusFound { - if err := budgets.output(budgets.owner(operationIndex), len(stored.Document.Payload)+128); err != nil { - setReadFailure(result, sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED, err, true) - continue - } - } applyReadResult(result, stored) } - return outcome, nil + return response, nil } diff --git a/internal/service/read_batching_test.go b/internal/service/read_batching_test.go index bc117c4..f4b6603 100644 --- a/internal/service/read_batching_test.go +++ b/internal/service/read_batching_test.go @@ -109,16 +109,13 @@ func TestReadMicrobatchSharesBoundedWorkingSet(t *testing.T) { if backend.reads.Load() != 1 { t.Fatalf("128 small RPCs used %d backend reads, want 1", backend.reads.Load()) } - if server.server.inFlightBytes != 0 || server.server.inFlightRequests != 0 { - t.Fatal("read admission leaked") - } } -func TestReadMicrobatchBoundsLargeSnapshotsAndRepeatedOutputs(t *testing.T) { +func TestReadMicrobatchUsesCollectedBatchForLargeSnapshotsAndRepeatedOutputs(t *testing.T) { for _, shared := range []bool{false, true} { t.Run(fmt.Sprint(shared), func(t *testing.T) { memoryStore := memory.New() - backend := &readCapacityStorage{Storage: memoryStore, maximum: 512} + backend := &readCapacityStorage{Storage: memoryStore} server := completionServer(t, backend) server.server.maxReadBytes = 512 calls := make([]*batchCall[*sink.ReadRequest, *sink.ReadResponse], 12) @@ -139,7 +136,7 @@ func TestReadMicrobatchBoundsLargeSnapshotsAndRepeatedOutputs(t *testing.T) { } responses = append(responses, result.response) } - if backend.reads.Load() != int64(len(calls)) { + if backend.reads.Load() != 1 { t.Fatalf("unexpected reads for large records: %d", backend.reads.Load()) } responses[0].Results[0].Document.Payload[0] = '!' @@ -150,13 +147,13 @@ func TestReadMicrobatchBoundsLargeSnapshotsAndRepeatedOutputs(t *testing.T) { } } -func TestReadMicrobatchRetainsCallerLimitsAndOrderAcrossShrinking(t *testing.T) { +func TestReadMicrobatchRetainsCallerResponseLimitsAndOrder(t *testing.T) { memoryStore := memory.New() for _, key := range []string{"a", "b", "c"} { seedReadCapacity(t, memoryStore, key, 100) } seedReadCapacity(t, memoryStore, "oversized", 700) - backend := &readCapacityStorage{Storage: memoryStore, maximum: 512} + backend := &readCapacityStorage{Storage: memoryStore} server := completionServer(t, backend) server.server.maxReadBytes = 512 calls := []*batchCall[*sink.ReadRequest, *sink.ReadResponse]{ @@ -190,72 +187,6 @@ func TestReadMicrobatchRetainsCallerLimitsAndOrderAcrossShrinking(t *testing.T) } } -func TestReadMicrobatchKeepsCompletedCallWhenLaterReadFails(t *testing.T) { - memoryStore := memory.New() - for _, key := range []string{"a", "b"} { - seedReadCapacity(t, memoryStore, key, 300) - } - backend := &readCapacityStorage{Storage: memoryStore, maximum: 512, failAt: 2} - server := completionServer(t, backend) - server.server.maxReadBytes = 512 - first := readCapacityCall(t.Context(), "a") - second := readCapacityCall(t.Context(), "b") - calls := []*batchCall[*sink.ReadRequest, *sink.ReadResponse]{first, second} - server.executeReads(t.Context(), calls) - completed := awaitCompletion(t, first.result) - failed := awaitCompletion(t, second.result) - if completed.err != nil || completed.response.Results[0].Status != sink.ReadStatus_READ_STATUS_FOUND { - t.Fatal(completed) - } - if status.Code(failed.err) != codes.Unavailable { - t.Fatal(failed.err) - } -} - -func TestReadBatchConcurrencyUsesStoreLimit(t *testing.T) { - backend := &readCapacityStorage{Storage: memory.New(), started: make(chan struct{}, 3), gate: make(chan struct{})} - core := completionServer(t, backend).server - core.maxInFlightRequests = 2 - opts := BatchingOptions{MaxOperations: 1, MaxWait: time.Millisecond} - server, err := NewBatchingServer(core, opts) - if err != nil { - t.Fatal(err) - } - defer server.Close() - defer close(backend.gate) - results := make(chan error, 3) - for index := range 3 { - call := readCapacityCall(t.Context(), fmt.Sprint(index)) - go func() { _, err := server.Read(t.Context(), call.request); results <- err }() - if index < 2 { - awaitCompletion(t, backend.started) - } - } - waitForQueuedCalls(t, server.reads, 1) - if backend.peak.Load() != 2 { - t.Fatalf("read concurrency = %d", backend.peak.Load()) - } - select { - case <-backend.started: - t.Fatal("third read exceeded the store limit") - default: - } - // Canceling all RPCs is separately covered; here release the I/O gate and - // verify queued requests can use the newly available capacity. - backend.gate <- struct{}{} - if err := awaitCompletion(t, results); err != nil { - t.Fatal(err) - } - awaitCompletion(t, backend.started) - backend.gate <- struct{}{} - backend.gate <- struct{}{} - for range 2 { - if err := awaitCompletion(t, results); err != nil { - t.Fatal(err) - } - } -} - func TestReadBatchCancellationKeepsOtherCallerAlive(t *testing.T) { backend := &readCapacityStorage{Storage: memory.New(), started: make(chan struct{}, 1), gate: make(chan struct{})} core := completionServer(t, backend).server diff --git a/internal/service/request_budgets.go b/internal/service/request_budgets.go index 58c6a8b..6fc2b4e 100644 --- a/internal/service/request_budgets.go +++ b/internal/service/request_budgets.go @@ -3,7 +3,6 @@ package service import ( "context" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/storage" ) @@ -13,10 +12,7 @@ import ( type requestBudgets struct { owners []int count int - producer *capacity.Scope trackers []*forwarding.Tracker - scopes []*capacity.Scope - contexts []context.Context } func (b *requestBudgets) callerCount() int { @@ -48,26 +44,6 @@ func (b *requestBudgets) add(count int) { b.count++ } -func sharedSnapshotBudget(owners []int, budgets []*storage.ReadBudget) *storage.ReadBudget { - if len(owners) == 1 { - return budgets[owners[0]] - } - if len(budgets) == 1 && len(owners) > 0 { - return budgets[0] - } - // Repeated operations do not add callers or need separate snapshot charges. - maximum := min(len(owners), len(budgets)) - shared := make([]*storage.ReadBudget, 0, maximum) - seen := make(map[int]bool, maximum) - for _, owner := range owners { - if !seen[owner] { - shared = append(shared, budgets[owner]) - seen[owner] = true - } - } - return storage.NewSharedReadBudget(shared) -} - func (b *requestBudgets) tracker(owner int) *forwarding.Tracker { if b == nil || owner >= len(b.trackers) { return nil @@ -77,40 +53,12 @@ func (b *requestBudgets) tracker(owner int) *forwarding.Tracker { func (b *requestBudgets) addContext(ctx context.Context, count int) { b.add(count) b.trackers = append(b.trackers, forwarding.FromContext(ctx)) - b.scopes = append(b.scopes, capacity.FromContext(ctx)) - b.contexts = append(b.contexts, ctx) } func contextBudgets(ctx context.Context, count int) *requestBudgets { - if forwarding.FromContext(ctx) == nil && capacity.FromContext(ctx) == nil { + if forwarding.FromContext(ctx) == nil { return nil } budgets := &requestBudgets{} budgets.addContext(ctx, count) return budgets } - -func (b *requestBudgets) output(owner, bytes int) error { - if b == nil || owner >= len(b.scopes) { - return nil - } - return b.scopes[owner].OutputFrom(b.contexts[owner], b.producer, bytes) -} - -func (b *requestBudgets) releaseOutput(owner, bytes int) { - if b == nil || owner >= len(b.scopes) { - return - } - b.scopes[owner].ReleaseOutputFrom(b.producer, bytes) -} - -func (b *requestBudgets) ownsInput() bool { - if b == nil || len(b.scopes) != b.count { - return false - } - for _, scope := range b.scopes { - if scope == nil { - return false - } - } - return true -} diff --git a/internal/service/request_budgets_benchmark_test.go b/internal/service/request_budgets_benchmark_test.go index ee497be..d44ad30 100644 --- a/internal/service/request_budgets_benchmark_test.go +++ b/internal/service/request_budgets_benchmark_test.go @@ -6,33 +6,9 @@ import ( sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/merge" - "github.com/liran/sink/internal/storage" "github.com/liran/sink/internal/storage/memory" ) -var benchmarkSnapshotBudget *storage.ReadBudget - -func BenchmarkSharedSnapshotBudget(b *testing.B) { - for _, callers := range []int{1, 128} { - for _, operations := range []int{1, 128, 1024} { - b.Run(fmt.Sprintf("callers=%d/operations=%d", callers, operations), func(b *testing.B) { - budgets := make([]*storage.ReadBudget, callers) - for i := range budgets { - budgets[i] = storage.NewReadBudget(4096) - } - owners := make([]int, operations) - for i := range owners { - owners[i] = i % callers - } - b.ReportAllocs() - for b.Loop() { - benchmarkSnapshotBudget = sharedSnapshotBudget(owners, budgets) - } - }) - } - } -} - func BenchmarkRepeatedRecordRead(b *testing.B) { for _, count := range []int{128, 1000} { b.Run(fmt.Sprintf("operations=%d", count), func(b *testing.B) { diff --git a/internal/service/reservation.go b/internal/service/reservation.go deleted file mode 100644 index 9a7cdd3..0000000 --- a/internal/service/reservation.go +++ /dev/null @@ -1,67 +0,0 @@ -package service - -import ( - "context" - "errors" - "github.com/liran/sink/internal/capacity" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// An admitted write can return unused document capacity while waiting for -// storage. Growth is atomic and nonblocking: waiting while retaining documents -// could deadlock several writers that all need their next working set. -type admissionReservation struct { - managed *capacity.Lease - ctx context.Context - pool *admissionPool - stores []string - bytes int - maximum int - released bool -} - -func (r *admissionReservation) resize(bytes int) error { - if r.managed != nil { - current := r.managed.Bytes() - if int64(bytes) < current { - r.managed.Shrink(current - int64(bytes)) - return nil - } - return r.managed.Grow(r.ctx, int64(bytes)-current, capacity.Response) - } - pool := r.pool - pool.admissionMu.Lock() - defer pool.admissionMu.Unlock() - if r.released || bytes < 0 || bytes > r.maximum { - return errors.New("invalid execution reservation resize") - } - delta := bytes - r.bytes - if delta == 0 { - return nil - } - if delta > 0 { - full := delta > pool.maxInFlightBytes-pool.inFlightBytes - for _, waiter := range pool.admissionWaiters { - if !pool.admissionSlotsFull(*waiter) { - full = true - break - } - } - if full { - store := pool.metrics.RequestStores(r.stores) - pool.metrics.ObserveAdmissionPoolRejected(store, pool.name, "resize") - return status.Error(codes.ResourceExhausted, "Sink execution capacity cannot restore the write working set") - } - } - r.bytes = bytes - pool.inFlightBytes += delta - pool.adjustStoreBytes(r.stores, delta) - store := pool.metrics.RequestStores(r.stores) - pool.metrics.AdjustAdmissionPool(store, pool.name, 0, delta) - if delta < 0 { - pool.wakeAdmissionWaiter() - } - return nil -} diff --git a/internal/service/scan_admission_test.go b/internal/service/scan_admission_test.go deleted file mode 100644 index 43436e3..0000000 --- a/internal/service/scan_admission_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package service - -import ( - "testing" - - "github.com/liran/sink/internal/merge" - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestScansLeaveOrdinaryRequestCapacity(t *testing.T) { - luaOpts := merge.LuaOptions{} - lua, err := merge.NewLuaEngine(luaOpts) - if err != nil { - t.Fatal(err) - } - opts := Options{BoundStore: "primary", Storage: memory.New(), Lua: lua} - s, err := New(opts) - if err != nil { - t.Fatal(err) - } - scan := admissionRequest{encodedBytes: (56 << 20) + 128, stores: []string{"mongo"}, scan: true} - for range 2 { - _, release, admitErr := s.admitRequest(t.Context(), scan) - if admitErr != nil { - t.Fatal(admitErr) - } - t.Cleanup(release) - } - _, _, err = s.admitRequest(t.Context(), scan) - if status.Code(err) != codes.ResourceExhausted { - t.Fatalf("third Mongo scan must hit scan quota: %v", err) - } - ordinary := admissionRequest{encodedBytes: (64 << 20) + 128, stores: []string{"mongo"}} - _, release, err := s.admitRequest(t.Context(), ordinary) - if err != nil { - t.Fatalf("scans starved ordinary Read/Merge: %v", err) - } - release() - if s.inFlightBytes != 2*scan.encodedBytes || s.scanBytes != s.inFlightBytes { - t.Fatal("ordinary release corrupted scan accounting") - } -} - -func TestScanRequestQuotaReleases(t *testing.T) { - luaOpts := merge.LuaOptions{} - lua, err := merge.NewLuaEngine(luaOpts) - if err != nil { - t.Fatal(err) - } - opts := Options{BoundStore: "primary", Storage: memory.New(), Lua: lua, MaxInFlightRequests: 4, - MaxInFlightBytes: 4096, MaxScanRequests: 2, MaxScanBytes: 2048} - s, err := New(opts) - if err != nil { - t.Fatal(err) - } - first := admissionRequest{encodedBytes: 100, scan: true, stores: []string{"a"}} - _, release, err := s.admitRequest(t.Context(), first) - if err != nil { - t.Fatal(err) - } - second := admissionRequest{encodedBytes: 100, scan: true, stores: []string{"b"}} - _, releaseSecond, err := s.admitRequest(t.Context(), second) - if err != nil { - t.Fatal(err) - } - third := admissionRequest{encodedBytes: 100, scan: true, stores: []string{"c"}} - _, _, err = s.admitRequest(t.Context(), third) - if status.Code(err) != codes.ResourceExhausted { - t.Fatalf("request quota: %v", err) - } - release() - releaseSecond() - if s.scanRequests != 0 || s.scanBytes != 0 { - t.Fatal("scan quota leaked") - } - _, release, err = s.admitRequest(t.Context(), first) - if err != nil { - t.Fatal(err) - } - release() -} diff --git a/internal/service/scan_wait_test.go b/internal/service/scan_wait_test.go deleted file mode 100644 index 929b8b6..0000000 --- a/internal/service/scan_wait_test.go +++ /dev/null @@ -1,239 +0,0 @@ -package service - -import ( - "context" - "testing" - "testing/synctest" - "time" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" - "google.golang.org/genproto/googleapis/rpc/errdetails" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type waitingScanStorage struct { - storage.Storage - storage.NativeStorage - calls int - block bool -} - -func (s *waitingScanStorage) Scan(ctx context.Context, _ storage.ScanRequest) (storage.ScanResponse, error) { - s.calls++ - response := storage.ScanResponse{} - if s.block { - <-ctx.Done() - return response, ctx.Err() - } - return response, nil -} - -func TestScanAdmissionAndExecutionSharePageDeadline(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - backend := &waitingScanStorage{Storage: memory.New(), block: true} - server := completionServer(t, backend).server - server.requestTimeout = 3 * time.Second - busy := admissionRequest{encodedBytes: server.maxInFlightBytes} - _, release, err := server.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - command := &sink.Command{Uri: "sink://primary", ContentType: "application/json", Payload: []byte(`{}`)} - request := &sink.ScanRequest{Command: command} - finished := make(chan error, 1) - started := time.Now() - go func() { _, err := server.Scan(t.Context(), request); finished <- err }() - synctest.Wait() - if backend.calls != 0 || len(server.admissionWaiters) != 1 { - t.Fatal("scan did not wait before backend execution") - } - time.Sleep(time.Second) - release() - synctest.Wait() - if backend.calls != 1 || len(server.admissionWaiters) != 0 { - t.Fatal("released capacity did not wake scan") - } - if err := <-finished; status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("page deadline: %v", err) - } - if elapsed := time.Since(started); elapsed != 3*time.Second { - t.Fatalf("admission reset the page deadline: %s", elapsed) - } - if server.inFlightBytes != 0 || server.scanBytes != 0 || server.inFlightRequests != 0 { - t.Fatal("deadline leaked execution capacity") - } - }) -} - -func TestScanAdmissionWaitIsBoundedAndCanceled(t *testing.T) { - for _, mode := range []string{"wait_timeout", "canceled", "page_timeout"} { - t.Run(mode, func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - backend := &waitingScanStorage{Storage: memory.New()} - server := completionServer(t, backend).server - busy := admissionRequest{encodedBytes: server.maxInFlightBytes} - _, release, err := server.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - if mode == "page_timeout" { - server.requestTimeout = time.Second - } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - command := &sink.Command{Uri: "sink://primary"} - request := &sink.ScanRequest{Command: command} - finished := make(chan error, 1) - go func() { _, err := server.Scan(ctx, request); finished <- err }() - synctest.Wait() - if mode == "canceled" { - cancel() - } - err = <-finished - want := codes.ResourceExhausted - if mode == "canceled" { - want = codes.Canceled - } else if mode == "page_timeout" { - want = codes.DeadlineExceeded - } - if status.Code(err) != want { - t.Fatalf("got %v, want %s", err, want) - } - if mode == "wait_timeout" { - assertScanAdmissionDetail(t, err, "wait_timeout") - } - if backend.calls != 0 || len(server.admissionWaiters) != 0 || server.inFlightBytes != busy.encodedBytes { - t.Fatal("failed scan executed or leaked a reservation") - } - release() - if _, err := server.Scan(t.Context(), request); err != nil || backend.calls != 1 { - t.Fatalf("scan did not recover: %v", err) - } - }) - }) - } -} - -func TestScanWaitQueueBounds(t *testing.T) { - for _, limit := range []string{"requests", "bytes"} { - t.Run(limit, func(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := completionServer(t, memory.New()).server - server.maxInFlightBytes = 100 - server.maxScanRequests = 10 - server.maxScanBytes = 100 - switch limit { - case "requests": - server.maxScanRequests = 1 - case "bytes": - server.maxScanBytes = 15 - } - busy := admissionRequest{encodedBytes: 100} - _, release, err := server.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - defer release() - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - scan := admissionRequest{encodedBytes: 10, scan: true, wait: true, stores: []string{"primary"}} - finished := make(chan error, 1) - go func() { _, _, err := server.admitRequest(ctx, scan); finished <- err }() - synctest.Wait() - _, _, err = server.admitRequest(t.Context(), scan) - assertScanAdmissionDetail(t, err, "queue") - if len(server.admissionWaiters) != 1 || server.inFlightBytes != 100 { - t.Fatal("queue cap did not bound retained scans") - } - cancel() - if err := <-finished; status.Code(err) != codes.Canceled { - t.Fatal(err) - } - synctest.Wait() - if len(server.admissionWaiters) != 0 { - t.Fatal("canceled scans leaked queue capacity") - } - }) - }) - } -} - -func TestOversizedScanDoesNotWaitOrAdvertiseRetry(t *testing.T) { - server := completionServer(t, memory.New()).server - server.maxInFlightBytes = 100 - server.maxScanBytes = 50 - for _, size := range []int{51, 101} { - scan := admissionRequest{encodedBytes: size, scan: true, wait: true, stores: []string{"primary"}} - _, _, err := server.admitRequest(t.Context(), scan) - failure := status.Convert(err) - if failure.Code() != codes.ResourceExhausted || len(failure.Details()) != 0 || len(server.admissionWaiters) != 0 { - t.Fatalf("permanent scan limit advertised retry or queued: %v", err) - } - } -} - -func TestScanWaitsBehindOlderWriteReservation(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := completionServer(t, memory.New()).server - server.maxInFlightBytes = 100 - busy := admissionRequest{encodedBytes: 20} - _, releaseBusy, err := server.admitRequest(t.Context(), busy) - if err != nil { - t.Fatal(err) - } - write := admissionRequest{encodedBytes: 95, wait: true} - admitted := make(chan context.CancelFunc, 1) - go func() { - _, release, err := server.admitRequest(t.Context(), write) - if err != nil { - t.Error(err) - } - admitted <- release - }() - synctest.Wait() - scan := admissionRequest{encodedBytes: 10, scan: true, wait: true, stores: []string{"primary"}} - scanned := make(chan context.CancelFunc, 1) - go func() { - _, release, err := server.admitRequest(t.Context(), scan) - if err != nil { - t.Error(err) - } - scanned <- release - }() - synctest.Wait() - if len(scanned) != 0 || len(server.admissionWaiters) != 2 { - t.Fatal("scan bypassed an older write despite fairness") - } - releaseBusy() - synctest.Wait() - releaseWrite := <-admitted - if releaseWrite == nil || len(scanned) != 0 { - t.Fatal("older write did not receive capacity first") - } - releaseWrite() - synctest.Wait() - releaseScan := <-scanned - if releaseScan == nil { - t.Fatal("scan failed instead of waiting for capacity") - } - releaseScan() - if server.inFlightBytes != 0 || server.scanBytes != 0 || len(server.admissionWaiters) != 0 { - t.Fatal("fair admission leaked reservations") - } - }) -} - -func assertScanAdmissionDetail(t *testing.T, err error, reason string) { - t.Helper() - failure := status.Convert(err) - if failure.Code() != codes.ResourceExhausted || len(failure.Details()) != 1 { - t.Fatalf("missing admission detail: %v", err) - } - detail, ok := failure.Details()[0].(*errdetails.ErrorInfo) - if !ok || detail.GetDomain() != "sink" || detail.GetReason() != "SCAN_ADMISSION_REJECTED" || detail.GetMetadata()["reason"] != reason { - t.Fatalf("wrong admission detail: %v", detail) - } -} diff --git a/internal/service/server.go b/internal/service/server.go index 8cb90e4..5155b0c 100644 --- a/internal/service/server.go +++ b/internal/service/server.go @@ -9,7 +9,6 @@ import ( "time" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/merge" sinkmetrics "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/protocol" @@ -22,9 +21,7 @@ import ( type Server struct { boundStore string - memory *capacity.Pool sink.UnimplementedSinkServer - *admissionPool storage storage.Storage lua *merge.LuaEngine @@ -33,9 +30,6 @@ type Server struct { maxMergeAttempts int metrics *sinkmetrics.Metrics maxReadBytes int - maxSnapshotBytes int - maxOutputBytes int - publishAdmission *admissionPool } func (s *Server) Write(ctx context.Context, req *sink.WriteRequest) (*sink.WriteResponse, error) { @@ -60,27 +54,7 @@ func (s *Server) write(ctx context.Context, req *sink.WriteRequest, budgets *req } observation := s.newWriteObservation(req) defer observation.finish() - estimate := s.estimateWriteExecution(req, budgets.callerCount(), returningCallerCount(req, budgets)) - reservation := &admissionReservation{} - admission := admissionRequest{encodedBytes: estimate.bytes, stores: operationStores(req.GetOperations()), wait: budgets != nil, reservation: reservation} - admission.inputBytes = req.SizeVT() + 128*len(req.GetOperations()) - admission.resultBytes = failureResponseBytes(len(req.GetOperations())) - admission.sharedInput = budgets.ownsInput() - admission.publish = req.GetCompletionMode() == sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED started := time.Now() - ctx, release, err := s.admitRequest(ctx, admission) - observation.phase("admission", started) - if err != nil { - return nil, err - } - defer release() - if budgets == nil { - budgets = contextBudgets(ctx, len(req.GetOperations())) - } - if budgets != nil { - budgets.producer = capacity.FromContext(ctx) - } - started = time.Now() luaPrograms, err := parseLuaPrograms(req.GetLuaPrograms()) if err != nil { observation.phase("parse", started) @@ -126,9 +100,7 @@ func (s *Server) write(ctx context.Context, req *sink.WriteRequest, budgets *req } groups := buildWriteGroups(operations) - memory := &writeMemoryReservation{reservation: reservation, estimate: estimate} executionOptions := writeExecutionOptions{ - memory: memory, returns: newWriteReturns(req, budgets, s.maxReadBytes), budgets: budgets, completion: completion, @@ -175,16 +147,6 @@ func (s *Server) delete(ctx context.Context, req *sink.DeleteRequest, budgets *r if !validCompletionMode(req.GetCompletionMode()) { return nil, status.Error(codes.InvalidArgument, "delete request has an invalid completion mode") } - admission := admissionRequest{encodedBytes: req.SizeVT() + failureResponseBytes(len(req.GetOperations())), stores: operationStores(req.GetOperations()), wait: budgets != nil} - admission.inputBytes = req.SizeVT() + 128*len(req.GetOperations()) - admission.resultBytes = failureResponseBytes(len(req.GetOperations())) - admission.sharedInput = budgets.ownsInput() - admission.publish = req.GetCompletionMode() == sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED - ctx, release, err := s.admitRequest(ctx, admission) - if err != nil { - return nil, err - } - defer release() response := &sink.DeleteResponse{ Results: make([]*sink.DeleteResult, len(req.GetOperations())), diff --git a/internal/service/server_test.go b/internal/service/server_test.go index 520df7e..e0410ba 100644 --- a/internal/service/server_test.go +++ b/internal/service/server_test.go @@ -17,7 +17,6 @@ import ( "github.com/liran/sink/internal/testuri" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/merge" sinkmetrics "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/queue" @@ -265,12 +264,7 @@ func TestMergeConflictMetricsRecordRetriesAndExhaustion(t *testing.T) { if err != nil { t.Fatalf("merge.NewLuaEngine() error = %v", err) } - memoryOptions := capacity.Options{Bytes: 256 << 20, BurstPercent: 10, WaitTimeout: 2 * time.Second} - pool, err := capacity.New(memoryOptions) - if err != nil { - t.Fatal(err) - } - options := service.Options{BoundStore: "primary", Memory: pool, + options := service.Options{BoundStore: "primary", Storage: conflictStorage{}, Lua: luaEngine, MaxMergeAttempts: 2, @@ -629,12 +623,7 @@ func newTestServer(t testing.TB, store storage.Storage, publisher queue.Publishe if err != nil { t.Fatalf("NewLuaEngine() error = %v", err) } - memoryOptions := capacity.Options{Bytes: 256 << 20, BurstPercent: 10, WaitTimeout: 2 * time.Second} - pool, err := capacity.New(memoryOptions) - if err != nil { - t.Fatal(err) - } - options := service.Options{BoundStore: "primary", Memory: pool, + options := service.Options{BoundStore: "primary", Storage: store, Lua: luaEngine, Publisher: publisher, diff --git a/internal/service/store_capacity_test.go b/internal/service/store_capacity_test.go deleted file mode 100644 index e60801a..0000000 --- a/internal/service/store_capacity_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package service - -import ( - "testing" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/storage/memory" -) - -func TestWriteBatchSplitsAtExecutionByteLimit(t *testing.T) { - backend := &syncCapacityStorage{Storage: memory.New()} - batching := completionServer(t, backend) - mode := sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED - first := completionWriteCall(t.Context(), mode, completionPut("first", 1)) - second := completionWriteCall(t.Context(), mode, completionPut("second", 1)) - limit := batching.server.estimateWriteExecution(second.request, 1, 0).bytes - batching.server.maxInFlightBytes = limit - calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{first, second} - batching.executeWrites(t.Context(), calls) - assertCompletionWrites(t, calls) - if backend.writes.Load() != 2 || batching.server.inFlightBytes != 0 { - t.Fatal("batch ignored store cap or leaked bytes") - } -} - -func TestReadBatchSplitsAtExecutionByteLimit(t *testing.T) { - backend := &syncCapacityStorage{Storage: memory.New()} - batching := completionServer(t, backend) - first := readCapacityCall(t.Context(), "first") - second := readCapacityCall(t.Context(), "second") - limit := 2*batching.server.maxReadBytes + second.request.SizeVT() + failureResponseBytes(1) - batching.server.maxInFlightBytes = limit - calls := []*batchCall[*sink.ReadRequest, *sink.ReadResponse]{first, second} - batching.executeReads(t.Context(), calls) - for _, call := range calls { - result := awaitCompletion(t, call.result) - if result.err != nil || len(result.response.GetResults()) != 1 { - t.Fatalf("store-capped read lost result: %v, %v", result.response, result.err) - } - } - if backend.reads.Load() != 2 || batching.server.inFlightBytes != 0 { - t.Fatal("read batch ignored store cap or leaked bytes") - } -} diff --git a/internal/service/sync_capacity_test.go b/internal/service/sync_capacity_test.go index 2c6ba49..4ff4b98 100644 --- a/internal/service/sync_capacity_test.go +++ b/internal/service/sync_capacity_test.go @@ -27,15 +27,11 @@ func TestSynchronousSmallMergesShareBoundedWorkingSet(t *testing.T) { if backend.reads.Load() != 1 || backend.writes.Load() != 1 { t.Fatalf("small records split by hypothetical document size: reads=%d writes=%d", backend.reads.Load(), backend.writes.Load()) } - if server.server.inFlightBytes != 0 || server.server.inFlightRequests != 0 { - t.Fatal("working capacity leaked") - } } -func TestReturnedPutsShareKnownDocumentReservation(t *testing.T) { +func TestReturnedPutsKeepCollectedBatch(t *testing.T) { backend := &syncCapacityStorage{Storage: memory.New()} server := completionServer(t, backend) - server.server.maxInFlightBytes = 128 << 20 calls := make([]*batchCall[*sink.WriteRequest, *sink.WriteResponse], 128) for index := range calls { operation := completionPut(fmt.Sprintf("returned-%d", index), index) @@ -50,12 +46,9 @@ func TestReturnedPutsShareKnownDocumentReservation(t *testing.T) { t.Fatalf("returned Put %d lost its document: %v, %v", index, result.response, result.err) } } - if backend.writes.Load() != 1 || server.server.inFlightBytes != 0 { - t.Fatalf("known small results fragmented a batch: writes=%d retained=%d", backend.writes.Load(), server.server.inFlightBytes) - } } -func TestSynchronousMergeStreamsLargeSnapshotsAndOutputs(t *testing.T) { +func TestSynchronousMergeProcessesCollectedSnapshotsAndOutputs(t *testing.T) { for _, scenario := range []string{"snapshots", "outputs", "conflicts", "returns"} { t.Run(scenario, func(t *testing.T) { memoryStore := memory.New() @@ -66,9 +59,6 @@ func TestSynchronousMergeStreamsLargeSnapshotsAndOutputs(t *testing.T) { server := completionServer(t, backend) server.server.maxReadBytes = 1024 - server.server.maxSnapshotBytes = 1024 - - server.server.maxOutputBytes = 1024 var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] for index := range 8 { key := fmt.Sprintf("record-%d", index) @@ -110,21 +100,18 @@ func TestSynchronousMergeStreamsLargeSnapshotsAndOutputs(t *testing.T) { t.Fatalf("%s attempted %d times, want %d", key, backend.attempts[key], want) } } - if backend.maxWriteBytes > server.server.maxReadBytes { - t.Fatalf("output chunk grew beyond working set: %d", backend.maxWriteBytes) + if backend.maxWriteBytes <= server.server.maxReadBytes { + t.Fatalf("batch was split by the response byte limit: %d", backend.maxWriteBytes) } }) } } -func TestSynchronousChunksKeepCallerQuotaAndSharedRecordIsolation(t *testing.T) { +func TestSynchronousMergesProcessEntireRequestAndSharedRecord(t *testing.T) { backend := memory.New() server := completionServer(t, backend) server.server.maxReadBytes = 400 - server.server.maxSnapshotBytes = 400 - - server.server.maxOutputBytes = 400 for _, key := range []string{"a", "b"} { address, err := protocol.ParseAddress(completionAddress(key)) if err != nil { @@ -138,13 +125,13 @@ func TestSynchronousChunksKeepCallerQuotaAndSharedRecordIsolation(t *testing.T) second := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, completionMerge("b", 1)) calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{first, second} server.executeWrites(t.Context(), calls) - oversized := awaitCompletion(t, first.result) + firstResult := awaitCompletion(t, first.result) healthy := awaitCompletion(t, second.result) - if oversized.err != nil || healthy.err != nil { - t.Fatalf("chunk errors: %v, %v", oversized.err, healthy.err) + if firstResult.err != nil || healthy.err != nil { + t.Fatalf("chunk errors: %v, %v", firstResult.err, healthy.err) } - if oversized.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED || oversized.response.Results[1].GetFailure().GetCode() != sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED { - t.Fatalf("original caller quota reset across chunks: %v", oversized.response) + if firstResult.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED || firstResult.response.Results[1].Status != sink.WriteStatus_WRITE_STATUS_APPLIED { + t.Fatalf("request was limited by an intermediate quota: %v", firstResult.response) } if healthy.response.Results[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED { t.Fatal(healthy.response) @@ -156,8 +143,8 @@ func TestSynchronousChunksKeepCallerQuotaAndSharedRecordIsolation(t *testing.T) operation := storage.ReadOperation{Address: address} request := storage.ReadRequest{Operations: []storage.ReadOperation{operation}} read, err := backend.Read(t.Context(), request) - if err != nil || string(read.Results[0].Document.Payload) != `{"value":1}` { - t.Fatalf("failed caller changed shared record: %v, %v", read, err) + if err != nil || string(read.Results[0].Document.Payload) != `{"value":101}` { + t.Fatalf("folded chain lost a caller: %v, %v", read, err) } } diff --git a/internal/service/working_memory_test.go b/internal/service/working_memory_test.go new file mode 100644 index 0000000..4229f38 --- /dev/null +++ b/internal/service/working_memory_test.go @@ -0,0 +1,54 @@ +package service + +import ( + "fmt" + "strings" + "testing" + + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/storage/memory" +) + +func TestCollectedMergesUseProcessMemoryInsteadOfDocumentQuotas(t *testing.T) { + for _, batched := range []bool{false, true} { + t.Run(fmt.Sprint(batched), func(t *testing.T) { + backend := &syncCapacityStorage{Storage: memory.New()} + server := completionServer(t, backend) + server.server.maxReadBytes = 1024 + var operations []*sink.WriteOperation + for index := range 8 { + operation := completionMerge(fmt.Sprint(index), 1) + operation.GetMerge().IncomingDocument.Payload = []byte(`{"value":1,"padding":"` + strings.Repeat("x", 4096) + `"}`) + operation.GetMerge().LuaProgram.Source = []byte(`return function(current, incoming) return incoming end`) + operations = append(operations, operation) + } + // Both the first outputs and the second round's snapshots/inputs + // exceed the response limit, including within one original RPC. + for range 2 { + if batched { + var calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse] + for start := 0; start < len(operations); start += 2 { + call := completionWriteCall(t.Context(), sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, operations[start:start+2]...) + calls = append(calls, call) + } + server.executeWrites(t.Context(), calls) + assertCompletionWrites(t, calls) + } else { + request := &sink.WriteRequest{Operations: operations, CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} + response, err := server.server.Write(t.Context(), request) + if err != nil { + t.Fatal(err) + } + for _, result := range response.Results { + if result.Status != sink.WriteStatus_WRITE_STATUS_APPLIED { + t.Fatal(result) + } + } + } + } + if backend.reads.Load() != 2 || backend.writes.Load() != 2 { + t.Fatalf("collected operations were split: reads=%d writes=%d", backend.reads.Load(), backend.writes.Load()) + } + }) + } +} diff --git a/internal/service/write.go b/internal/service/write.go index 94b03de..98f34d6 100644 --- a/internal/service/write.go +++ b/internal/service/write.go @@ -11,8 +11,6 @@ import ( "github.com/liran/sink/internal/protocol" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/merge" "github.com/liran/sink/internal/storage" "google.golang.org/grpc/codes" @@ -58,7 +56,6 @@ type writeExecutionOptions struct { observation *writeObservation budgets *requestBudgets WaitUntilVisible bool - memory *writeMemoryReservation } func (o writeExecutionOptions) complete(group writeGroup, results []*sink.WriteResult) { @@ -369,94 +366,31 @@ func (s *Server) executeWriteAttempt( results []*sink.WriteResult, opts writeExecutionOptions, ) ([]writeGroup, error) { - attempt := writeAttempt{ - options: opts, - results: results, - snapshots: opts.budgets.fresh(forwarding.Snapshots, s.maxSnapshotBytes), - inputs: opts.budgets.fresh(forwarding.Inputs, s.maxReadBytes), - outputs: opts.budgets.fresh(forwarding.Outputs, s.maxOutputBytes), - output: storage.NewReadBudget(s.maxOutputBytes), + attempt := writeAttempt{options: opts, results: results} + if err := contextError(ctx); err != nil { + return nil, err } - next := make([]writeGroup, 0) - pending := groups - limit := len(groups) - for len(pending) > 0 { - if err := contextError(ctx); err != nil { - return nil, err - } - if err := opts.memory.reservation.resize(opts.memory.estimate.bytes); err != nil { - // These groups have not written in this attempt. Previous successful - // chunks keep their results; only the remaining operations may retry. - for _, group := range pending { - for _, operation := range group.operations { - setWriteFailure(results[operation.index], sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED, err, true) - } - opts.complete(group, results) - } - return next, nil - } - count := min(limit, len(pending)) - batch := pending[:count] - read, err := s.readWriteSnapshots(ctx, batch, &attempt) - if err != nil { - return nil, err - } - attempt.snapshotBytes = 0 - for _, snapshot := range read.Results { - attempt.snapshotBytes += len(snapshot.Document.Payload) + 128 - } - outcome, err := s.applyWriteSnapshots(ctx, batch, read.Results, &attempt) - clear(read.Results) - if err != nil { - return nil, err - } - next = append(next, outcome.conflicts...) - pending = pending[count:] - if len(outcome.deferred) > 0 { - // Subsequent reads use the observed chunk size instead of repeatedly - // fetching a large tail that cannot fit in the retained working set. - limit = max(1, count-len(outcome.deferred)) - pending = append(outcome.deferred, pending...) - } + read, err := s.readWriteSnapshots(ctx, groups, &attempt) + if err != nil { + return nil, err } - return next, nil + defer clear(read.Results) + return s.applyWriteSnapshots(ctx, groups, read.Results, &attempt) } type writeAttempt struct { - options writeExecutionOptions - results []*sink.WriteResult - snapshots []*storage.ReadBudget - inputs []*storage.ReadBudget - outputs []*storage.ReadBudget - output *storage.ReadBudget - snapshotBytes int - pendingCandidateBytes int - finalCommit bool -} - -type writeChunkOutcome struct { - conflicts []writeGroup - deferred []writeGroup + options writeExecutionOptions + results []*sink.WriteResult } func (s *Server) readWriteSnapshots(ctx context.Context, groups []writeGroup, attempt *writeAttempt) (storage.ReadResponse, error) { - working := storage.NewReadBudget(s.maxSnapshotBytes) + budget := storage.NewUnboundedReadBudget() readOperations := make([]storage.ReadOperation, 0, len(groups)) for _, group := range groups { - operation := group.operations[0] - owners := make([]int, 0, len(group.operations)) - for _, member := range group.operations { - owners = append(owners, attempt.options.budgets.owner(member.index)) - } - budget := sharedSnapshotBudget(owners, attempt.snapshots) - if attempt.options.budgets.callerCount() > 1 { - budget = storage.NewWorkingSetReadBudget(budget, working) - } - budget = storage.WithMemoryBudget(ctx, budget, attempt.options.memory.reservation.managed) - readOperation := storage.ReadOperation{Address: operation.address, Budget: budget} + readOperation := storage.ReadOperation{Address: group.operations[0].address, Budget: budget} readOperations = append(readOperations, readOperation) } - readRequest := storage.ReadRequest{Operations: readOperations, Budget: storage.NewReadBudget(s.maxSnapshotBytes)} + readRequest := storage.ReadRequest{Operations: readOperations, Budget: budget} started := time.Now() readResponse, err := s.storage.Read(ctx, readRequest) attempt.options.observation.phase("storage_read", started) @@ -469,82 +403,27 @@ func (s *Server) readWriteSnapshots(ctx context.Context, groups []writeGroup, at return readResponse, nil } -func (s *Server) applyWriteSnapshots(ctx context.Context, groups []writeGroup, snapshots []storage.ReadResult, attempt *writeAttempt) (writeChunkOutcome, error) { - var outcome writeChunkOutcome +func (s *Server) applyWriteSnapshots(ctx context.Context, groups []writeGroup, snapshots []storage.ReadResult, attempt *writeAttempt) ([]writeGroup, error) { opts := attempt.options results := attempt.results candidates := make([]writeGroupCandidate, 0, len(groups)) - outputBytes := 0 for index, stored := range snapshots { group := groups[index] - if errors.Is(stored.Err, storage.ErrReadWorkingSetFull) { - outcome.deferred = append(outcome.deferred, group) - continue - } input := writeGroupPreparation{group: group, stored: stored, results: results} - var candidate writeGroupCandidate - var include bool - if opts.budgets == nil { - candidate, include = s.prepareWriteGroup(ctx, input) - } else { - preparation := batchedWritePreparation{writeGroupPreparation: input, budgets: opts.budgets, inputs: attempt.inputs, outputs: attempt.outputs} - candidate, include = s.prepareBatchedWriteGroup(ctx, preparation) - } + candidate, include := s.prepareWriteGroup(ctx, input) if include { - candidateBytes := len(candidate.operation.Document.Payload) + 128 - growthErr := capacity.Grow(ctx, opts.memory.reservation.managed, 3*candidateBytes) - if growthErr != nil { - failure := storage.WriteResult{Status: storage.WriteStatusFailed, Err: storage.ResourceExhaustedError(growthErr)} - applyWriteGroupResult(group, results, failure) - opts.complete(group, results) - continue - } if err := opts.returns.reserve(group, candidate.operation.Document); err != nil { - if opts.memory.reservation.managed != nil { - opts.memory.reservation.managed.Shrink(3 * int64(candidateBytes)) - } failure := storage.WriteResult{Status: storage.WriteStatusFailed, Err: err} applyWriteGroupResult(group, results, failure) opts.complete(group, results) continue } - if opts.budgets == nil { - if err := attempt.output.Reserve(len(candidate.operation.Document.Payload)); err != nil { - if opts.memory.reservation.managed != nil { - opts.memory.reservation.managed.Shrink(3 * int64(candidateBytes)) - } - failure := storage.WriteResult{Status: storage.WriteStatusFailed, Err: err} - applyWriteGroupResult(group, results, failure) - opts.complete(group, results) - continue - } - } - if opts.budgets.callerCount() > 1 && len(candidates) > 0 && candidateBytes > s.maxOutputBytes-outputBytes { - attempt.pendingCandidateBytes = candidateBytes - conflicts, err := s.commitWriteCandidates(ctx, candidates, attempt) - attempt.pendingCandidateBytes = 0 - if err != nil { - return outcome, err - } - outcome.conflicts = append(outcome.conflicts, conflicts...) - if opts.memory.reservation.managed != nil { - opts.memory.reservation.managed.Shrink(3 * int64(outputBytes)) - } - clear(candidates) - candidates = candidates[:0] - outputBytes = 0 - } candidates = append(candidates, candidate) - outputBytes += candidateBytes } else { opts.complete(group, results) } } - attempt.finalCommit = true - conflicts, err := s.commitWriteCandidates(ctx, candidates, attempt) - attempt.finalCommit = false - outcome.conflicts = append(outcome.conflicts, conflicts...) - return outcome, err + return s.commitWriteCandidates(ctx, candidates, attempt) } func (s *Server) commitWriteCandidates(ctx context.Context, candidates []writeGroupCandidate, attempt *writeAttempt) ([]writeGroup, error) { @@ -554,27 +433,6 @@ func (s *Server) commitWriteCandidates(ctx context.Context, candidates []writeGr if err := contextError(ctx); err != nil { return nil, err } - if attempt.finalCommit || attempt.options.memory.reservation.managed != nil { - candidateBytes := 0 - for _, candidate := range candidates { - candidateBytes += len(candidate.operation.Document.Payload) + 128 - } - // All candidates in this chunk are now known. Keep the still-live - // snapshot slice, output and copy allowance; return unused capacity - // throughout the storage call, including refresh=wait_for latency. - if err := attempt.options.memory.retain(attempt.snapshotBytes, candidateBytes+attempt.pendingCandidateBytes); err != nil { - if attempt.options.memory.reservation.managed == nil { - return nil, err - } - for _, candidate := range candidates { - failure := storage.WriteResult{Status: storage.WriteStatusFailed, Err: storage.ResourceExhaustedError(err)} - applyWriteGroupResult(candidate.group, attempt.results, failure) - attempt.options.complete(candidate.group, attempt.results) - } - return nil, nil - } - } - writeOperations := make([]storage.WriteOperation, 0, len(candidates)) for _, candidate := range candidates { writeOperations = append(writeOperations, candidate.operation) diff --git a/internal/service/write_completion_test.go b/internal/service/write_completion_test.go index 1eadec2..f3c90a1 100644 --- a/internal/service/write_completion_test.go +++ b/internal/service/write_completion_test.go @@ -200,12 +200,6 @@ func TestWriteCompletionReleasesDocumentBeforeWholeRPC(t *testing.T) { if err != nil || response.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_APPLIED { t.Fatalf("completed document remains owned by unrelated unfinished operation: %v, %v", response, err) } - core.admissionMu.Lock() - requests, reserved := core.inFlightRequests, core.inFlightBytes - core.admissionMu.Unlock() - if requests < 1 || reserved <= 0 { - t.Fatal("early completion released capacity still owned by unfinished execution") - } backend.unblock() if err := awaitCompletion(t, done); err != nil { t.Fatal(err) diff --git a/internal/service/write_group.go b/internal/service/write_group.go index 7e7a9f1..477e56b 100644 --- a/internal/service/write_group.go +++ b/internal/service/write_group.go @@ -94,52 +94,3 @@ func applyWriteGroupResult(group writeGroup, results []*sink.WriteResult, stored applyWriteResult(result, stored) } } - -type batchedWritePreparation struct { - writeGroupPreparation - budgets *requestBudgets - inputs []*storage.ReadBudget - outputs []*storage.ReadBudget -} - -func (s *Server) prepareBatchedWriteGroup(ctx context.Context, input batchedWritePreparation) (writeGroupCandidate, bool) { - candidate := writeGroupCandidate{group: input.group} - current := input.stored - include := false - operations := input.group.operations - for start := 0; start < len(operations); { - owner := input.budgets.owner(operations[start].index) - end := start + 1 - for end < len(operations) && input.budgets.owner(operations[end].index) == owner { - end++ - } - part := writeGroup{operations: operations[start:end]} - start = end - if current.Status == storage.ReadStatusFound { - if err := input.inputs[owner].Reserve(len(current.Document.Payload)); err != nil { - failure := storage.WriteResult{Status: storage.WriteStatusFailed, Err: err} - applyWriteGroupResult(part, input.results, failure) - continue - } - } - preparation := writeGroupPreparation{group: part, stored: current, results: input.results} - prepared, ok := s.prepareWriteGroup(ctx, preparation) - if !ok { - continue - } - if err := input.outputs[owner].Reserve(len(prepared.operation.Document.Payload)); err != nil { - failure := storage.WriteResult{Status: storage.WriteStatusFailed, Err: err} - applyWriteGroupResult(part, input.results, failure) - continue - } - if !include { - candidate.operation = prepared.operation - include = true - } else { - candidate.operation.Document = prepared.operation.Document - } - current.Status = storage.ReadStatusFound - current.Document = prepared.operation.Document - } - return candidate, include -} diff --git a/internal/service/write_memory.go b/internal/service/write_memory.go deleted file mode 100644 index 12345e7..0000000 --- a/internal/service/write_memory.go +++ /dev/null @@ -1,19 +0,0 @@ -package service - -type writeMemoryReservation struct { - reservation *admissionReservation - estimate writeExecutionEstimate -} - -func (m *writeMemoryReservation) retain(snapshotBytes int, candidateBytes int) error { - if m.reservation.managed != nil { - return m.reservation.resize(2*snapshotBytes + 3*candidateBytes) - } - // This refines the existing payload reservation, not the process RSS limit. - // Keep all original input, Lua source and returned-document allowances. - // The snapshots remain live until applyWriteSnapshots returns. Allow an - // extra snapshot copy and three candidate copies for conversion/encoding. - working := min(m.estimate.workingBytes, 2*snapshotBytes+3*candidateBytes) - bytes := m.estimate.bytes - m.estimate.workingBytes + working - return m.reservation.resize(bytes) -} diff --git a/internal/service/write_memory_cancellation_test.go b/internal/service/write_memory_cancellation_test.go deleted file mode 100644 index 2ac3d6e..0000000 --- a/internal/service/write_memory_cancellation_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package service - -import ( - "context" - "testing" - "time" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/capacity" -) - -func TestCanceledReturnReleasesCandidateBeforeSibling(t *testing.T) { - backend := newHeldReadStorage(t) - server := completionServer(t, backend) - options := capacity.Options{Bytes: 192 << 10, BurstPercent: 10, WaitTimeout: 20 * time.Millisecond} - pool, err := capacity.New(options) - if err != nil { - t.Fatal(err) - } - server.server.memory = pool - firstScope, secondScope := pool.NewScope(), pool.NewScope() - defer firstScope.Release() - defer secondScope.Release() - firstContext, cancel := context.WithCancel(capacity.WithScope(t.Context(), firstScope)) - defer cancel() - secondContext := capacity.WithScope(t.Context(), secondScope) - source := []byte(`return function(current, incoming) local value = "x"; for i=1,15 do value=value..value end; return {value=value} end`) - first, second := completionMerge("canceled", 1), completionMerge("healthy", 1) - first.GetMerge().GetLuaProgram().Source = source - second.GetMerge().GetLuaProgram().Source = source - first.ReturnDocument = true - mode := sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED - firstCall := completionWriteCall(firstContext, mode, first) - secondCall := completionWriteCall(secondContext, mode, second) - calls := []*batchCall[*sink.WriteRequest, *sink.WriteResponse]{firstCall, secondCall} - done := make(chan struct{}) - go func() { server.executeWrites(t.Context(), calls); close(done) }() - awaitCompletion(t, backend.entered) - cancel() - backend.unblock() - awaitCompletion(t, done) - firstResult, secondResult := awaitCompletion(t, firstCall.result), awaitCompletion(t, secondCall.result) - t.Logf("canceled: %v error=%v", firstResult.response, firstResult.err) - t.Logf("healthy: %v error=%v", secondResult.response, secondResult.err) - if secondResult.err != nil || secondResult.response.GetResults()[0].Status != sink.WriteStatus_WRITE_STATUS_APPLIED { - t.Fatal("discarded candidate reservation poisoned healthy sibling") - } -} diff --git a/internal/service/write_memory_test.go b/internal/service/write_memory_test.go deleted file mode 100644 index 92e37e9..0000000 --- a/internal/service/write_memory_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package service - -import ( - "github.com/liran/sink/internal/testuri" - - "context" - "testing" - - sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/storage" - "github.com/liran/sink/internal/storage/memory" - "go.mongodb.org/mongo-driver/v2/bson" -) - -type phaseMemoryStorage struct { - storage.Storage - storage.NativeStorage - core *Server - entered chan storage.WriteRequest - resume chan bool - readReservations []int - writes int -} - -func (s *phaseMemoryStorage) Read(ctx context.Context, req storage.ReadRequest) (storage.ReadResponse, error) { - s.core.admissionMu.Lock() - s.readReservations = append(s.readReservations, s.core.inFlightBytes) - s.core.admissionMu.Unlock() - return s.Storage.Read(ctx, req) -} - -func (s *phaseMemoryStorage) Write(ctx context.Context, req storage.WriteRequest) (storage.WriteResponse, error) { - s.writes++ - s.entered <- req - response := storage.WriteResponse{} - select { - case conflict := <-s.resume: - if conflict { - for range req.Operations { - result := storage.WriteResult{Status: storage.WriteStatusPreconditionFailed} - response.Results = append(response.Results, result) - } - return response, nil - } - case <-ctx.Done(): - return response, ctx.Err() - } - return s.Storage.Write(ctx, req) -} - -func (s *phaseMemoryStorage) Scan(context.Context, storage.ScanRequest) (storage.ScanResponse, error) { - response := storage.ScanResponse{} - return response, nil -} - -type phaseWriteResult struct { - response *sink.WriteResponse - err error -} - -func TestSmallVisibleWriteReturnsUnusedWorkingCapacityToScans(t *testing.T) { - backend := &phaseMemoryStorage{Storage: memory.New(), entered: make(chan storage.WriteRequest, 1), resume: make(chan bool)} - defer close(backend.resume) - core := completionServer(t, backend).server - backend.core = core - core.maxInFlightBytes = 70 << 20 - core.maxScanBytes = 64 << 20 - operation := completionMerge("record", 1) - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, Operations: []*sink.WriteOperation{operation}} - estimate := core.estimateWriteExecution(request, 1, 0) - finished := make(chan phaseWriteResult, 1) - go func() { - response, err := core.Write(t.Context(), request) - result := phaseWriteResult{response: response, err: err} - finished <- result - }() - write := <-backend.entered - if !write.WaitUntilVisible { - t.Fatal("visible completion was weakened") - } - core.admissionMu.Lock() - retained := core.inFlightBytes - storeRetained := core.inFlightBytes - core.admissionMu.Unlock() - if retained >= 1<<20 || retained <= 0 || retained != storeRetained { - t.Fatalf("small write kept hypothetical snapshots: global=%d store=%d", retained, storeRetained) - } - command := bson.D{{Key: "find", Value: "products"}} - payload, err := bson.Marshal(command) - if err != nil { - t.Fatal(err) - } - native := &sink.Command{Uri: "sink://primary/catalog", ContentType: "application/bson", Payload: payload} - scan := &sink.ScanRequest{Command: native} - if _, err := core.Scan(t.Context(), scan); err != nil { - t.Fatalf("56 MiB BSON scan could not use released write capacity: %v", err) - } - if len(finished) != 0 { - t.Fatal("write returned before storage visibility completed") - } - backend.resume <- false - result := <-finished - if result.err != nil || result.response.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_APPLIED { - t.Fatalf("write result: %v, %v", result.response, result.err) - } - if core.inFlightBytes != 0 || core.inFlightRequests != 0 { - t.Fatal("resized reservation leaked or released its original size twice") - } - t.Logf("synthetic small merge reservation: %d -> %d bytes while waiting for visibility", estimate.bytes, retained) -} - -func TestWriteRestoresWorkingCapacityBeforeConflictRetry(t *testing.T) { - backend := &phaseMemoryStorage{Storage: memory.New(), entered: make(chan storage.WriteRequest, 1), resume: make(chan bool)} - defer close(backend.resume) - core := completionServer(t, backend).server - backend.core = core - operation := completionMerge("record", 1) - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, Operations: []*sink.WriteOperation{operation}} - peak := core.estimateWriteExecution(request, 1, 0).bytes - finished := make(chan phaseWriteResult, 1) - go func() { - response, err := core.Write(t.Context(), request) - result := phaseWriteResult{response: response, err: err} - finished <- result - }() - <-backend.entered - backend.resume <- true - <-backend.entered - if len(backend.readReservations) != 2 || backend.readReservations[0] != peak || backend.readReservations[1] != peak { - t.Fatalf("retry read lacked peak protection: %v want %d", backend.readReservations, peak) - } - backend.resume <- false - result := <-finished - if result.err != nil || result.response.GetResults()[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_APPLIED || core.inFlightBytes != 0 { - t.Fatalf("retry lost result or capacity: %v, %v", result.response, result.err) - } -} - -func TestWriteCannotRegrowWithoutCapacityAndPreservesEarlierSuccess(t *testing.T) { - backend := &phaseMemoryStorage{Storage: memory.New(), entered: make(chan storage.WriteRequest, 1), resume: make(chan bool)} - defer close(backend.resume) - core := completionServer(t, backend).server - backend.core = core - core.maxInFlightBytes = 70 << 20 - put := completionPut("successful", 1) - conditional := completionMerge("conflict", 1) - request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE, Operations: []*sink.WriteOperation{put, conditional}} - finished := make(chan phaseWriteResult, 1) - go func() { - response, err := core.Write(t.Context(), request) - result := phaseWriteResult{response: response, err: err} - finished <- result - }() - <-backend.entered - backend.resume <- false - <-backend.entered - pressure := admissionRequest{encodedBytes: 65 << 20} - _, release, err := core.admitRequest(t.Context(), pressure) - if err != nil { - t.Fatal(err) - } - defer release() - backend.resume <- true - result := <-finished - if result.err != nil || len(result.response.GetResults()) != 2 { - t.Fatalf("lost partial response: %v, %v", result.response, result.err) - } - if result.response.Results[0].GetStatus() != sink.WriteStatus_WRITE_STATUS_APPLIED { - t.Fatal("earlier successful put was changed") - } - failure := result.response.Results[1].GetFailure() - if failure.GetCode() != sink.FailureCode_FAILURE_CODE_RESOURCE_EXHAUSTED || !failure.GetRetryable() { - t.Fatalf("unapplied conflict must report safe per-operation retry: %v", failure) - } - if len(backend.readReservations) != 1 || backend.writes != 2 || core.inFlightBytes != pressure.encodedBytes { - t.Fatal("restoring capacity allocated, replayed successful work or leaked its reservation") - } -} - -func (s *phaseMemoryStorage) BatchKey(address storage.Address) (string, error) { - return testuri.BatchKey(address) -} diff --git a/internal/service/write_observation_test.go b/internal/service/write_observation_test.go index b8920e5..1f2fe42 100644 --- a/internal/service/write_observation_test.go +++ b/internal/service/write_observation_test.go @@ -37,7 +37,6 @@ func TestWriteObservationsCountOnlyRetriedDocuments(t *testing.T) { `sink_write_phase_duration_seconds_count{phase="storage_read",store="primary"} 2`, `sink_write_phase_duration_seconds_count{phase="storage_write_applied",store="primary"} 2`, `sink_write_phase_duration_seconds_count{phase="lua",store="primary"} 3`, - `sink_write_phase_duration_seconds_count{phase="admission",store="primary"} 1`, `sink_write_execution_rounds_sum{phase="storage_read",store="primary"} 2`, `sink_write_execution_rounds_count{phase="storage_write",store="primary"} 1`, } @@ -177,8 +176,6 @@ func TestBatchedWritesKeepQueueAndPhaseMetricsSeparateByStore(t *testing.T) { fmt.Sprintf(`sink_batcher_queued_bytes{method="Write",store="%s"} 0`, store), fmt.Sprintf(`sink_write_phase_duration_seconds_count{phase="storage_write_applied",store="%s"} %d`, store, count), fmt.Sprintf(`sink_write_execution_rounds_sum{phase="storage_write",store="%s"} %d`, store, count), - fmt.Sprintf(`sink_admission_pool_requests{pool="execution",store="%s"} 0`, store), - fmt.Sprintf(`sink_admission_pool_bytes{pool="execution",store="%s"} 0`, store), } for _, line := range wanted { if !strings.Contains(body, line) { diff --git a/internal/service/write_return.go b/internal/service/write_return.go index 552dd2b..b3849bd 100644 --- a/internal/service/write_return.go +++ b/internal/service/write_return.go @@ -62,9 +62,6 @@ func (r *writeReturns) reserve(group writeGroup, document storage.Document) erro if difference > r.remaining[owner] { return storage.ResourceExhaustedError(errors.New("returned write documents exceed response byte budget")) } - if err := r.owners.output(owner, difference); err != nil { - return storage.ResourceExhaustedError(err) - } r.remaining[owner] -= difference r.owners.tracker(owner).Observe(forwarding.Returns, r.maximum[owner]-r.remaining[owner]) r.sizes[operation.index] = max(charge, r.sizes[operation.index]) @@ -97,7 +94,6 @@ func (r *writeReturns) settle(group writeGroup, results []*sink.WriteResult) { owner := r.owners.owner(operation.index) unused := max(0, reserved-retained) r.remaining[owner] += unused - r.owners.releaseOutput(owner, unused) delete(r.sizes, operation.index) } } diff --git a/internal/storage/mongodb/memory.go b/internal/storage/mongodb/memory.go deleted file mode 100644 index 0a3c6e2..0000000 --- a/internal/storage/mongodb/memory.go +++ /dev/null @@ -1,20 +0,0 @@ -package mongodb - -import ( - "context" - - "github.com/liran/sink/internal/capacity" -) - -// The driver allocates a complete wire message before exposing cursor.Current. -// This opaque allowance exists only while reading from MongoDB; document copies -// are acquired separately at their actual sizes. It is not an RPC admission fee. -func acquireWireMemory(ctx context.Context) (*capacity.Lease, error) { - lease := capacity.FromContext(ctx).NewLease() - lease.MarkOpaque() - if err := capacity.Grow(ctx, lease, 48<<20); err != nil { - capacity.Close(lease) - return nil, err - } - return lease, nil -} diff --git a/internal/storage/mongodb/native.go b/internal/storage/mongodb/native.go index f6879ba..e0e5d7e 100644 --- a/internal/storage/mongodb/native.go +++ b/internal/storage/mongodb/native.go @@ -10,7 +10,6 @@ import ( "time" "github.com/liran/sink-go/uri" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" @@ -113,11 +112,6 @@ func (s *Store) validateNativeCommand(req storage.NativeRequest, scan bool) (str func (s *Store) Execute(ctx context.Context, req storage.NativeRequest) (storage.NativeResponse, error) { var empty storage.NativeResponse - wire, err := acquireWireMemory(ctx) - if err != nil { - return empty, storage.ResourceExhaustedError(err) - } - defer capacity.Close(wire) database, command, err := s.validateNativeCommand(req, false) if err != nil { return empty, storage.InvalidArgumentError(err) @@ -143,7 +137,7 @@ func (s *Store) Execute(ctx context.Context, req storage.NativeRequest) (storage } return empty, storage.BackendError(err) } - budget := storage.NewResponseBudget(ctx, req.MaxBytes) + budget := storage.NewReadBudget(req.MaxBytes) if budgetErr := budget.Reserve(len(raw)); budgetErr != nil { return empty, budgetErr } @@ -192,10 +186,8 @@ func (s *Store) scanDocuments(ctx context.Context, req storage.ScanRequest, send }() batch := make([]storage.Document, 0, req.BatchSize) // Count consumes each page synchronously; these documents never become RPC - // output. Reuse a working-set lease after the visitor discards a page. - lease := capacity.FromContext(ctx).NewLease() - defer capacity.Close(lease) - budget := storage.WithMemoryBudget(ctx, storage.NewReadBudget(req.Request.MaxBytes), lease) + // output. Discard each page before reading the next. + budget := storage.NewReadBudget(req.Request.MaxBytes) for cursor.Next(ctx) { if err := budget.Reserve(len(cursor.Current)); err != nil { if len(batch) == 0 { @@ -204,11 +196,8 @@ func (s *Store) scanDocuments(ctx context.Context, req storage.ScanRequest, send if err := send(batch); err != nil { return err } - if lease != nil { - lease.Shrink(lease.Bytes()) - } batch = make([]storage.Document, 0, req.BatchSize) - budget = storage.WithMemoryBudget(ctx, storage.NewReadBudget(req.Request.MaxBytes), lease) + budget = storage.NewReadBudget(req.Request.MaxBytes) if err := budget.Reserve(len(cursor.Current)); err != nil { return err } @@ -219,11 +208,8 @@ func (s *Store) scanDocuments(ctx context.Context, req storage.ScanRequest, send if err := send(batch); err != nil { return err } - if lease != nil { - lease.Shrink(lease.Bytes()) - } batch = make([]storage.Document, 0, req.BatchSize) - budget = storage.WithMemoryBudget(ctx, storage.NewReadBudget(req.Request.MaxBytes), lease) + budget = storage.NewReadBudget(req.Request.MaxBytes) } } if err := cursor.Err(); err != nil { diff --git a/internal/storage/mongodb/query.go b/internal/storage/mongodb/query.go index 72969bd..3dfda46 100644 --- a/internal/storage/mongodb/query.go +++ b/internal/storage/mongodb/query.go @@ -7,7 +7,6 @@ import ( "fmt" "time" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo/options" @@ -32,11 +31,6 @@ func appendQueryStages(command bson.D, stages bson.A) (bson.D, error) { func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.QueryResponse, error) { var empty storage.QueryResponse - wire, err := acquireWireMemory(ctx) - if err != nil { - return empty, storage.ResourceExhaustedError(err) - } - defer capacity.Close(wire) if err := req.Validate(); err != nil { return empty, err } @@ -110,7 +104,7 @@ func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.Qu _ = cursor.Close(cleanup) }() result := storage.QueryResponse{} - budget := storage.NewResponseBudget(ctx, req.Request.MaxBytes) + budget := storage.NewReadBudget(req.Request.MaxBytes) for cursor.Next(ctx) { // Lookahead proves HasMore but is never retained or returned to the // caller, so a large following document must not reject this page. @@ -203,11 +197,6 @@ func canEstimateCount(command bson.D) bool { func (s *Store) Count(ctx context.Context, req storage.CountRequest) (storage.CountResponse, error) { var empty storage.CountResponse - wire, err := acquireWireMemory(ctx) - if err != nil { - return empty, storage.ResourceExhaustedError(err) - } - defer capacity.Close(wire) request := req.Request database, command, err := s.validateNativeCommand(request, true) if err != nil { diff --git a/internal/storage/mongodb/read.go b/internal/storage/mongodb/read.go index 2dbe3e2..12def68 100644 --- a/internal/storage/mongodb/read.go +++ b/internal/storage/mongodb/read.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo/options" @@ -91,12 +90,6 @@ func (s *Store) Read(ctx context.Context, req storage.ReadRequest) (storage.Read } func (s *Store) readGroup(ctx context.Context, group *readGroup, results []storage.ReadResult) { - wire, err := acquireWireMemory(ctx) - if err != nil { - s.setReadGroupError(group, results, storage.ResourceExhaustedError(err)) - return - } - defer capacity.Close(wire) ids := make([]any, 0, len(group.operations)) indexesByID := make(map[string][]int, len(group.operations)) budgets := make(map[int]*storage.ReadBudget, len(group.operations)) diff --git a/internal/storage/mongodb/scan.go b/internal/storage/mongodb/scan.go index ad750d5..adf463a 100644 --- a/internal/storage/mongodb/scan.go +++ b/internal/storage/mongodb/scan.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" "go.mongodb.org/mongo-driver/v2/bson" ) @@ -135,11 +134,6 @@ func scanFind(command bson.D, req storage.ScanRequest, position []byte) (bson.D, func (s *Store) Scan(ctx context.Context, req storage.ScanRequest) (storage.ScanResponse, error) { var empty storage.ScanResponse - wire, err := acquireWireMemory(ctx) - if err != nil { - return empty, storage.ResourceExhaustedError(err) - } - defer capacity.Close(wire) seek, err := req.Resume() if err != nil { return empty, err @@ -162,7 +156,7 @@ func (s *Store) Scan(ctx context.Context, req storage.ScanRequest) (storage.Scan _ = cursor.Close(cleanup) }() documents := make([]storage.Document, 0, req.BatchSize) - budget := storage.NewResponseBudget(ctx, req.Request.MaxBytes) + budget := storage.NewReadBudget(req.Request.MaxBytes) var position []byte more := false for cursor.Next(ctx) { diff --git a/internal/storage/read_budget.go b/internal/storage/read_budget.go index ae9968b..a6122a5 100644 --- a/internal/storage/read_budget.go +++ b/internal/storage/read_budget.go @@ -1,31 +1,20 @@ package storage import ( - "context" "errors" - "github.com/liran/sink/internal/capacity" "sync" ) const DefaultMaxReadBytes = 32 << 20 -// ErrReadWorkingSetFull asks the synchronous executor to read this record in a -// later chunk. It is never a failure of the original RPC's document budget. -var ErrReadWorkingSetFull = errors.New("read working set is full") - // ReadBudget is shared across every store and repeated key in one read. Reserve // before copying a result so small key batches cannot allocate unbounded output. type ReadBudget struct { mu sync.Mutex remaining int maximum int - shared []*ReadBudget - caller *ReadBudget - working *ReadBudget observe func(int) - memory *capacity.Lease - context context.Context - output *capacity.Scope + unlimited bool } func NewReadBudget(maxBytes int) *ReadBudget { @@ -43,91 +32,20 @@ func NewTrackedReadBudget(maxBytes int, observe func(int)) *ReadBudget { return budget } -// NewSharedReadBudget charges each interested caller once for a shared snapshot. -// A caller that exhausted its own quota cannot prevent another caller's read. -func NewSharedReadBudget(budgets []*ReadBudget) *ReadBudget { - if len(budgets) == 1 { - return budgets[0] - } - budget := &ReadBudget{shared: budgets} - return budget -} - -// NewWorkingSetReadBudget limits retained snapshots across independent RPCs. -// A deferred snapshot does not consume its caller's quota. The working budget -// must be an ordinary NewReadBudget, not another composite budget. -func NewWorkingSetReadBudget(caller, working *ReadBudget) *ReadBudget { - budget := &ReadBudget{caller: caller, working: working} - return budget -} - -func WithMemoryBudget(ctx context.Context, budget *ReadBudget, lease *capacity.Lease) *ReadBudget { - if lease == nil { - return budget - } - managed := &ReadBudget{caller: budget, memory: lease, context: ctx} - return managed -} - -func NewResponseBudget(ctx context.Context, maximum int) *ReadBudget { - budget := NewReadBudget(maximum) - if scope := capacity.FromContext(ctx); scope != nil { - managed := &ReadBudget{caller: budget, context: ctx, output: scope} - return managed - } +// NewUnboundedReadBudget leaves intermediate documents to process admission. +// Public response allowances continue to use NewReadBudget. +func NewUnboundedReadBudget() *ReadBudget { + budget := &ReadBudget{unlimited: true} return budget } func (b *ReadBudget) Reserve(size int) error { - if b.output != nil { - if err := b.caller.Reserve(size); err != nil { - return err - } - if err := b.output.Output(b.context, size+128); err != nil { - return ResourceExhaustedError(err) + if b.unlimited { + if size < 0 { + return ResourceExhaustedError(errors.New("invalid document size")) } return nil } - if b.memory != nil { - if err := b.caller.Reserve(size); err != nil { - return err - } - if err := capacity.Grow(b.context, b.memory, size+128); err != nil { - return ResourceExhaustedError(err) - } - return nil - } - if b.working != nil { - if err := b.working.Reserve(size); err != nil { - if size >= 0 && size <= b.working.maximum-128 { - return ErrReadWorkingSetFull - } - return err - } - if err := b.caller.Reserve(size); err != nil { - b.working.mu.Lock() - b.working.remaining += size + 128 - b.working.mu.Unlock() - return err - } - return nil - } - if len(b.shared) > 0 { - accepted := false - var failure error - for _, budget := range b.shared { - if err := budget.Reserve(size); err != nil { - failure = err - } else { - accepted = true - } - } - if accepted { - return nil - } - return failure - } - b.mu.Lock() defer b.mu.Unlock() // Include room for the per-operation protobuf envelope and revision. diff --git a/internal/storage/read_budget_test.go b/internal/storage/read_budget_test.go index 2486d19..3a1cd59 100644 --- a/internal/storage/read_budget_test.go +++ b/internal/storage/read_budget_test.go @@ -1,59 +1,21 @@ package storage import ( - "errors" "sync" "sync/atomic" "testing" ) -func TestWorkingSetBudgetDefersWithoutChargingCaller(t *testing.T) { - caller := NewReadBudget(512) - working := NewReadBudget(256) - budget := NewWorkingSetReadBudget(caller, working) - if err := budget.Reserve(100); err != nil { - t.Fatal(err) - } - if err := budget.Reserve(100); !errors.Is(err, ErrReadWorkingSetFull) { - t.Fatalf("full working set: %v", err) - } - next := NewWorkingSetReadBudget(caller, NewReadBudget(256)) - if err := next.Reserve(100); err != nil { - t.Fatalf("deferred read consumed caller quota: %v", err) - } - if err := NewWorkingSetReadBudget(caller, NewReadBudget(256)).Reserve(100); err == nil || errors.Is(err, ErrReadWorkingSetFull) { - t.Fatalf("caller quota was not enforced across chunks: %v", err) - } -} - -func TestWorkingSetBudgetRefundsRejectedCallerAndRejectsOversizedRecord(t *testing.T) { - working := NewReadBudget(256) - oversized := NewWorkingSetReadBudget(NewReadBudget(512), working) - if err := oversized.Reserve(256); err == nil || errors.Is(err, ErrReadWorkingSetFull) { - t.Fatalf("oversized record would be deferred forever: %v", err) - } - rejected := NewWorkingSetReadBudget(NewReadBudget(128), working) - if err := rejected.Reserve(100); err == nil { - t.Fatal("invalid caller was admitted") - } - healthy := NewWorkingSetReadBudget(NewReadBudget(256), working) - if err := healthy.Reserve(100); err != nil { - t.Fatalf("failed caller retained working capacity: %v", err) - } -} - -func TestWorkingSetBudgetIsSharedAcrossConcurrentReaders(t *testing.T) { +func TestReadResponseBudgetIsSharedAcrossConcurrentReaders(t *testing.T) { working := NewReadBudget(8 * 256) var accepted atomic.Int64 var readers sync.WaitGroup for range 64 { readers.Go(func() { - budget := NewWorkingSetReadBudget(NewReadBudget(256), working) + budget := working err := budget.Reserve(128) if err == nil { accepted.Add(1) - } else if !errors.Is(err, ErrReadWorkingSetFull) { - t.Error(err) } }) } diff --git a/internal/storage/search/client.go b/internal/storage/search/client.go index 711752d..1f00002 100644 --- a/internal/storage/search/client.go +++ b/internal/storage/search/client.go @@ -6,12 +6,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "strings" "time" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -35,11 +35,9 @@ type apiResponse struct { statusCode int body []byte headers http.Header - memory *capacity.Lease } func (r *apiResponse) close() { - capacity.Close(r.memory) r.body = nil } @@ -140,20 +138,17 @@ func (s *Store) performOnce(ctx context.Context, opts requestOptions, endpoint * if opts.maxBytes > 0 { maximum = min(maximum, opts.maxBytes) } - lease := capacity.FromContext(ctx).NewLease() - body, err := capacity.ReadAll(ctx, httpResponse.Body, maximum+1, lease) + body, err := io.ReadAll(io.LimitReader(httpResponse.Body, maximum+1)) if err != nil { - capacity.Close(lease) if status.Code(err) == codes.ResourceExhausted { return empty, storage.ResourceExhaustedError(err) } return empty, storage.BackendError(err) } if int64(len(body)) > maximum { - capacity.Close(lease) return empty, fmt.Errorf("%w: %s maximum is %d bytes", errResponseTooLarge, s.driver, maximum) } - response := apiResponse{statusCode: httpResponse.StatusCode, body: body, headers: httpResponse.Header.Clone(), memory: lease} + response := apiResponse{statusCode: httpResponse.StatusCode, body: body, headers: httpResponse.Header.Clone()} return response, nil } diff --git a/internal/storage/search/memory_lifetime_test.go b/internal/storage/search/memory_lifetime_test.go deleted file mode 100644 index f7160af..0000000 --- a/internal/storage/search/memory_lifetime_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package search - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/liran/sink/internal/capacity" - "github.com/liran/sink/internal/storage" -) - -func TestSplitReadsReleaseDiscardedHTTPBodies(t *testing.T) { - payload := `{"value":"` + strings.Repeat("x", 4000) + `"}` - var calls atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - var request multiGetRequest - if err := json.NewDecoder(r.Body).Decode(&request); err != nil { - t.Error(err) - return - } - documents := make([]string, 0, len(request.Documents)) - for _, document := range request.Documents { - documents = append(documents, fmt.Sprintf(`{"_index":%q,"_id":%q,"found":true,"_seq_no":0,"_primary_term":1,"_source":%s}`, document.Index, document.ID, payload)) - } - fmt.Fprintf(w, `{"docs":[%s]}`, strings.Join(documents, ",")) - }) - backend := httptest.NewServer(handler) - defer backend.Close() - opts := Options{Driver: DriverOpenSearch, Store: "primary", Endpoints: []string{backend.URL}, MaxResponseSize: 16 << 10} - store, err := New(opts) - if err != nil { - t.Fatal(err) - } - defer store.Close() - poolOpts := capacity.Options{Bytes: 192 << 10, BurstPercent: 10, WaitTimeout: 10 * time.Millisecond} - pool, err := capacity.New(poolOpts) - if err != nil { - t.Fatal(err) - } - scope := pool.NewScope() - defer scope.Release() - ctx := capacity.WithScope(t.Context(), scope) - snapshots := scope.NewLease() - budget := storage.WithMemoryBudget(ctx, storage.NewReadBudget(1<<20), snapshots) - request := storage.ReadRequest{Budget: budget} - for index := range 8 { - operation := storage.ReadOperation{Address: testAddress(fmt.Sprint(index))} - request.Operations = append(request.Operations, operation) - } - response, err := store.Read(ctx, request) - if err != nil { - t.Fatal(err) - } - found := 0 - for _, result := range response.Results { - if result.Status == storage.ReadStatusFound { - found++ - } else { - t.Logf("failed: %v", result.Err) - } - } - t.Logf("found=%d/8 requests=%d pool_used=%d actual_payload=%d", found, calls.Load(), pool.Used(), len(payload)*found) - if pool.Used() != snapshots.Bytes() { - t.Fatalf("HTTP working buffers retained after decode: %d", pool.Used()) - } - scope.Release() - request.Budget = storage.NewReadBudget(1 << 20) - store.maxResponseSize = 64 << 10 - controlScope := pool.NewScope() - defer controlScope.Release() - controlCtx := capacity.WithScope(t.Context(), controlScope) - control, controlErr := store.Read(controlCtx, request) - controlFound := 0 - for _, result := range control.Results { - if result.Status == storage.ReadStatusFound { - controlFound++ - } - } - t.Logf("control with same pool and documents, unsplit response: found=%d/8 error=%v", controlFound, controlErr) - if controlErr != nil || controlFound != 8 { - t.Fatal("invalid control") - } - if found != 8 { - t.Fatal("discarded oversized bodies consumed capacity needed by split reads") - } -} - -func TestLocalMemoryPressureDoesNotCoolDownSearchEndpoints(t *testing.T) { - var requests atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - fmt.Fprint(w, strings.Repeat("x", 24000)) - }) - first, second := httptest.NewServer(handler), httptest.NewServer(handler) - defer first.Close() - defer second.Close() - opts := Options{Driver: DriverOpenSearch, Store: "primary", Endpoints: []string{first.URL, second.URL}} - store, err := New(opts) - if err != nil { - t.Fatal(err) - } - defer store.Close() - poolOpts := capacity.Options{Bytes: 32 << 10, BurstPercent: 10, WaitTimeout: 10 * time.Millisecond} - pool, err := capacity.New(poolOpts) - if err != nil { - t.Fatal(err) - } - scope := pool.NewScope() - defer scope.Release() - ctx := capacity.WithScope(t.Context(), scope) - request := requestOptions{method: http.MethodGet, path: "/", retrySafe: true} - _, err = store.perform(ctx, request) - code, _ := storage.ErrorDetails(err) - cooled := 0 - for _, endpoint := range store.endpoints { - if endpoint.retryAfter.Load() > time.Now().UnixNano() { - cooled++ - } - } - t.Logf("error=%v code=%v requests=%d cooled=%d", err, code, requests.Load(), cooled) - if code != storage.ErrorCodeResourceExhausted || cooled != 0 || requests.Load() != 1 { - t.Fatal("local memory exhaustion retried and marked healthy endpoints unavailable") - } -} - -func TestSearchFailoverReleasesConsumedResponseMemory(t *testing.T) { - payload := strings.Repeat("x", 24000) - failed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusServiceUnavailable) - fmt.Fprint(w, payload) - })) - defer failed.Close() - healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - fmt.Fprint(w, payload) - })) - defer healthy.Close() - opts := Options{Driver: DriverOpenSearch, Store: "primary", Endpoints: []string{failed.URL, healthy.URL}} - store, err := New(opts) - if err != nil { - t.Fatal(err) - } - defer store.Close() - options := capacity.Options{Bytes: 96 << 10, BurstPercent: 10, WaitTimeout: time.Second} - pool, err := capacity.New(options) - if err != nil { - t.Fatal(err) - } - scope := pool.NewScope() - defer scope.Release() - ctx := capacity.WithScope(t.Context(), scope) - if err := store.Ping(ctx); err != nil { - t.Fatalf("discarded failover response consumed the next attempt's capacity: %v", err) - } - if pool.Used() != 0 { - t.Fatalf("completed ping retained HTTP response capacity: %d", pool.Used()) - } -} diff --git a/internal/storage/search/native.go b/internal/storage/search/native.go index 415f259..662817d 100644 --- a/internal/storage/search/native.go +++ b/internal/storage/search/native.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/liran/sink-go/uri" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" "golang.org/x/net/http/httpguts" ) @@ -116,7 +115,6 @@ func (s *Store) Execute(ctx context.Context, req storage.NativeRequest) (storage } type scanPage struct { - memory *capacity.Lease ScrollID string `json:"_scroll_id"` Hits *scanHits `json:"hits"` TimedOut *bool `json:"timed_out"` diff --git a/internal/storage/search/query.go b/internal/storage/search/query.go index 3dd8f45..78459c2 100644 --- a/internal/storage/search/query.go +++ b/internal/storage/search/query.go @@ -11,7 +11,6 @@ import ( "strings" "unicode/utf8" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" ) @@ -104,8 +103,6 @@ func (s *Store) performQuery(ctx context.Context, opts requestOptions) (scanPage return page, errors.New("search query returned incomplete cluster results") } } - page.memory = response.memory - response.memory = nil return page, nil } @@ -162,7 +159,6 @@ func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.Qu if err != nil { return empty, err } - defer capacity.Close(page.memory) if len(page.Hits.Hits) > req.PageSize { return empty, errors.New("search query exceeded its requested result count") } @@ -171,7 +167,7 @@ func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.Qu return empty, err } result := storage.QueryResponse{HasMore: hasMore} - budget := storage.NewResponseBudget(ctx, req.Request.MaxBytes) + budget := storage.NewReadBudget(req.Request.MaxBytes) for _, hit := range page.Hits.Hits { if err := budget.Reserve(len(hit)); err != nil { return empty, err @@ -233,7 +229,6 @@ func (s *Store) Count(ctx context.Context, req storage.CountRequest) (storage.Co if err != nil { return empty, err } - defer capacity.Close(page.memory) var total struct { Value *int64 `json:"value"` Relation string `json:"relation"` diff --git a/internal/storage/search/read.go b/internal/storage/search/read.go index 2b7d930..4ebc713 100644 --- a/internal/storage/search/read.go +++ b/internal/storage/search/read.go @@ -8,7 +8,6 @@ import ( "fmt" "net/http" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" ) @@ -38,7 +37,6 @@ type multiGetDocument struct { } type multiGetResponse struct { - memory *capacity.Lease Documents []multiGetDocument `json:"docs"` } @@ -88,7 +86,6 @@ func (s *Store) Read(ctx context.Context, req storage.ReadRequest) (storage.Read } applyMultiGetDocument(result, document, budget) } - capacity.Close(documents.memory) } return response, nil } @@ -137,8 +134,6 @@ func (s *Store) multiGet(ctx context.Context, works []readWork, source bool) (mu return empty, fmt.Errorf("search multi-get result %d has a missing index or mismatched document ID", index) } } - decoded.memory = response.memory - response.memory = nil return decoded, nil } diff --git a/internal/storage/search/read_budget_test.go b/internal/storage/search/read_budget_test.go index 26cf30e..3614c7f 100644 --- a/internal/storage/search/read_budget_test.go +++ b/internal/storage/search/read_budget_test.go @@ -24,7 +24,7 @@ func TestReadFailuresPreserveDocumentBudget(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - for _, scope := range []string{"request", "operation", "working set", "exhausted"} { + for _, scope := range []string{"request", "operation", "exhausted"} { t.Run(scope, func(t *testing.T) { const source = `{"value":1}` const charge = len(source) + 128 @@ -42,14 +42,10 @@ func TestReadFailuresPreserveDocumentBudget(t *testing.T) { Operations: []storage.ReadOperation{{Address: testAddress("bad")}, {Address: testAddress("good")}}, Budget: budget, } - if scope == "operation" || scope == "working set" { + if scope == "operation" { request.Budget = storage.NewTrackedReadBudget(0, nil) - working := storage.NewReadBudget(charge) for index := range request.Operations { request.Operations[index].Budget = budget - if scope == "working set" { - request.Operations[index].Budget = storage.NewWorkingSetReadBudget(budget, working) - } } } response, err := store.Read(t.Context(), request) diff --git a/internal/storage/search/scan.go b/internal/storage/search/scan.go index 558f0bc..8ed2f66 100644 --- a/internal/storage/search/scan.go +++ b/internal/storage/search/scan.go @@ -9,7 +9,6 @@ import ( "fmt" "strings" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" ) @@ -114,12 +113,11 @@ func (s *Store) Scan(ctx context.Context, req storage.ScanRequest) (storage.Scan if err != nil { return empty, err } - defer capacity.Close(page.memory) if len(page.Hits.Hits) > pageSize+1 { return empty, errors.New("search Scan exceeded its requested result count") } documents := make([]storage.Document, 0, req.BatchSize) - budget := storage.NewResponseBudget(ctx, req.Request.MaxBytes) + budget := storage.NewReadBudget(req.Request.MaxBytes) previous := seek.Position var position []byte more := false diff --git a/internal/storage/search/write.go b/internal/storage/search/write.go index 8731bd2..bfd58f1 100644 --- a/internal/storage/search/write.go +++ b/internal/storage/search/write.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" - "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/storage" ) @@ -190,7 +189,6 @@ func (s *Store) checkExistingWrites(ctx context.Context, works []readWork, check // Replace needs only existence and revision. Even metadata can exceed the // response cap for many long IDs, so split this read before any bulk write. documents, err := s.multiGet(ctx, works, false) - defer capacity.Close(documents.memory) if errors.Is(err, errResponseTooLarge) && len(works) > 1 && ctx.Err() == nil { middle := len(works) / 2 s.checkExistingWrites(ctx, works[:middle], check) diff --git a/internal/worker/capacity_test.go b/internal/worker/capacity_test.go index 583d9e8..679fde4 100644 --- a/internal/worker/capacity_test.go +++ b/internal/worker/capacity_test.go @@ -39,15 +39,14 @@ func (a *capacityApplier) Delete(ctx context.Context, req *sink.DeleteRequest) ( return a.Applier.Delete(ctx, req) } -func TestProcessorSplitsRejectedWritesAndDeletesWithoutReplayingSuccess(t *testing.T) { +func TestProcessorHandlesCollectedWritesAndDeletesWithoutReplayingSuccess(t *testing.T) { store := memory.New() luaOpts := merge.LuaOptions{} engine, err := merge.NewLuaEngine(luaOpts) if err != nil { t.Fatal(err) } - // Include room for one bounded failure response, while still forcing splits. - opts := service.Options{BoundStore: "primary", Storage: store, Lua: engine, MaxInFlightBytes: 2048} + opts := service.Options{BoundStore: "primary", Storage: store, Lua: engine} core, err := service.New(opts) if err != nil { t.Fatal(err) @@ -77,27 +76,19 @@ func TestProcessorSplitsRejectedWritesAndDeletesWithoutReplayingSuccess(t *testi } } for _, sizes := range [][]int{observed.writeSizes, observed.deleteSizes} { - if len(sizes) != 7 || sizes[0] != 4 { - t.Fatalf("unexpected split sizes %v", sizes) - } - singles := 0 - for _, count := range sizes { - if count == 1 { - singles++ - } - } - if singles != 4 { - t.Fatalf("successful records replayed: %v", sizes) + if len(sizes) != 1 || sizes[0] != 4 { + t.Fatalf("collected batch was split or replayed: %v", sizes) } } + } -func TestProcessorDoesNotSplitAmbiguousOrCanceledRequests(t *testing.T) { - for _, code := range []codes.Code{codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Internal, codes.ResourceExhausted} { +func TestProcessorDoesNotSplitFailedOrCanceledRequests(t *testing.T) { + for _, code := range []codes.Code{codes.Unknown, codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Internal, codes.ResourceExhausted} { t.Run(code.String(), func(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - if code == codes.ResourceExhausted { + if code == codes.Canceled { cancel() } applier := &capacityApplier{failure: status.Error(code, "injected failure")} diff --git a/internal/worker/failure_classification_test.go b/internal/worker/failure_classification_test.go index ec2f5dd..5a9371c 100644 --- a/internal/worker/failure_classification_test.go +++ b/internal/worker/failure_classification_test.go @@ -83,3 +83,54 @@ func assertFailureBarrier(t *testing.T, failure *sink.Failure, retain bool) { t.Fatalf("permanent failure blocked following valid record: calls=%d results=%v", applier.calls, results) } } + +type incompleteApplier struct { + write *sink.WriteResponse + deleted *sink.DeleteResponse +} + +func (a *incompleteApplier) Write(context.Context, *sink.WriteRequest) (*sink.WriteResponse, error) { + return a.write, nil +} +func (a *incompleteApplier) Delete(context.Context, *sink.DeleteRequest) (*sink.DeleteResponse, error) { + return a.deleted, nil +} + +func TestIncompleteOrCanceledResultsRemainRetryable(t *testing.T) { + if _, err := NewProcessor(nil); err == nil { + t.Fatal("missing applier accepted") + } + key := uri.StringKey("record") + address := &sink.RecordAddress{Uri: testuri.Record("primary", []string{"catalog", "records"}, key)} + document := &sink.Document{Encoding: sink.DocumentEncoding_DOCUMENT_ENCODING_JSON, Payload: []byte(`{}`)} + put := &sink.PutOperation{Document: document, Mode: sink.WriteMode_WRITE_MODE_UPSERT} + action := &sink.WriteOperation_Put{Put: put} + write := &sink.WriteOperation{Address: address, Action: action} + deleted := &sink.DeleteOperation{Address: address} + writeMutation, deleteMutation := queue.Mutation{Write: write}, queue.Mutation{Delete: deleted} + for _, canceled := range []bool{false, true} { + applier := &incompleteApplier{} + ctx, cancel := context.WithCancel(t.Context()) + if canceled { + // A permanent-looking result received after cancellation cannot prove rejection. + failure := &sink.Failure{Code: sink.FailureCode_FAILURE_CODE_INVALID_ARGUMENT, Retryable: false} + written := &sink.WriteResult{Status: sink.WriteStatus_WRITE_STATUS_FAILED, Failure: failure} + removed := &sink.DeleteResult{Status: sink.DeleteStatus_DELETE_STATUS_FAILED, Failure: failure} + applier.write = &sink.WriteResponse{Results: []*sink.WriteResult{written}} + applier.deleted = &sink.DeleteResponse{Results: []*sink.DeleteResult{removed}} + cancel() + } + processor, err := NewProcessor(applier) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []queue.Mutation{writeMutation, deleteMutation} { + err := processor.Handle(ctx, mutation) + var failure *ApplyError + if !errors.As(err, &failure) || !failure.Retryable() { + t.Fatalf("lost unresolved source record: %v", err) + } + } + cancel() + } +} diff --git a/internal/worker/processor.go b/internal/worker/processor.go index e11cbe5..c11f9c1 100644 --- a/internal/worker/processor.go +++ b/internal/worker/processor.go @@ -8,12 +8,10 @@ import ( sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/queue" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) -// Applier returns a top-level ResourceExhausted only when the entire request -// was rejected before execution. Per-operation failures may have applied work. +// Applier executes one collected mutation group. Errors keep the source records +// retryable; the processor never splits or replays a request after a top-level error. type Applier interface { Write(ctx context.Context, req *sink.WriteRequest) (*sink.WriteResponse, error) Delete(ctx context.Context, req *sink.DeleteRequest) (*sink.DeleteResponse, error) @@ -112,14 +110,6 @@ func (p *Processor) applyWrites(ctx context.Context, operations []mutationWork, } response, err := p.applier.Write(ctx, request) if err != nil { - if status.Code(err) == codes.ResourceExhausted && len(operations) > 1 && ctx.Err() == nil { - // Core admission rejected every operation. Smaller requests can fit - // without replaying an ambiguous write or a successful sibling. - middle := len(operations) / 2 - p.applyWrites(ctx, operations[:middle], results) - p.applyWrites(ctx, operations[middle:], results) - return - } for _, operation := range operations { results[operation.index] = NewApplyError("apply queued writes", true, err) } @@ -159,12 +149,6 @@ func (p *Processor) applyDeletes(ctx context.Context, operations []mutationWork, } response, err := p.applier.Delete(ctx, request) if err != nil { - if status.Code(err) == codes.ResourceExhausted && len(operations) > 1 && ctx.Err() == nil { - middle := len(operations) / 2 - p.applyDeletes(ctx, operations[:middle], results) - p.applyDeletes(ctx, operations[middle:], results) - return - } for _, operation := range operations { results[operation.index] = NewApplyError("apply queued deletes", true, err) } diff --git a/proto/forward/forward.proto b/proto/forward/forward.proto index fd030b7..91c3083 100644 --- a/proto/forward/forward.proto +++ b/proto/forward/forward.proto @@ -13,11 +13,11 @@ service Engine { rpc ForwardStream(ForwardRequest) returns (stream ResponseFrame); } -// Independent logical document budgets, including per-record envelope charges. +// Public response allowance, including per-record envelope charges. +// Intermediate allocations are owned by each process memory pool. message Budget { - uint64 snapshots = 1; - uint64 inputs = 2; - uint64 outputs = 3; + reserved 1, 2, 3; + reserved "snapshots", "inputs", "outputs"; uint64 returns = 4; } From 0a1d3c60119687c01cb7d043e1227a66b4ac2518 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 07:00:59 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=93=9D=20docs:=20align=20write=20flow?= =?UTF-8?q?=20and=20qualification=20with=20memory=20watermarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- docs/document-write-flow.md | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54fdf67..adf483a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@f9a2683b2946f6f3109c98d2c0a6ef3890a0745a + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@5b364be341f1bc0bca6050f67b6d20893ee760f0 with: - suite_ref: f9a2683b2946f6f3109c98d2c0a6ef3890a0745a + suite_ref: 5b364be341f1bc0bca6050f67b6d20893ee760f0 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/docs/document-write-flow.md b/docs/document-write-flow.md index deb54e4..e4ac65b 100644 --- a/docs/document-write-flow.md +++ b/docs/document-write-flow.md @@ -17,12 +17,12 @@ flowchart TD B --> Q[Gateway routes each Store group to its Engine] Q --> C{Requested completion_mode} C -->|WAIT_UNTIL_APPLIED / WAIT_UNTIL_VISIBLE| D[Engine in-memory batching] - D --> E[Write core: validation, admission control, execution] + D --> E[Write core: validation and execution] E --> F[Bound Store storage adapter] F --> G[MongoDB or Elasticsearch / OpenSearch] G --> H[Backend acknowledges; VISIBLE also waits for search visibility] H --> I[Client receives APPLIED] - C -->|RETURN_AFTER_ACCEPTED| J[Write core: validation, admission control, publish original operation] + C -->|RETURN_AFTER_ACCEPTED| J[Write core: validation and publish original operation] J --> K[Kafka topic for this store acknowledges acceptance] K --> L[Client receives ACCEPTED] K --> M[Worker consumes the operation] @@ -85,15 +85,15 @@ also take time. Gateway splits RPCs that span multiple Stores into single-Store groups and reassembles their results. Each Engine executes only its own Store group. -**Step 3: The write core parses operations and checks execution capacity.** +**Step 3: The write core parses the admitted operations.** -The core validates addresses, document encodings and payloads, and write -actions. It limits in-flight requests and execution bytes for this Engine. Direct -synchronous requests wait in a bounded admission queue; full queues and requests -that cannot fit receive `RESOURCE_EXHAUSTED`. Dispatched micro-batches also wait -within their deadlines. -Coalesced RPCs retain separate read/output budgets, and execution groups split -at RPC boundaries when their combined reservations exceed the byte limit. +Gateway and Engine check process memory watermarks before admitting new work. +The core validates addresses, document encodings, payloads and write actions. +Engine's method queues bound waiting operations and encoded input bytes. Dispatched +batches run as collected, without snapshot/output byte quotas or per-allocation +reservations. Each original RPC retains its returned-document allowance derived +from the gRPC send limit. Caller cancellation still stops work when no live caller +remains; memory pressure alone never cancels an admitted batch. In this example, Upsert means "write this complete document without requiring that the record already exist or be absent." It does not execute Lua or @@ -331,8 +331,7 @@ refresh wait. The two modes can execute concurrently for disjoint addresses when capacity permits. A mode change on the same address preserves batch collection order by completing the preceding run first. Write/Delete queues track these record dependencies across batches as well: later independent RPCs -can execute while an earlier batch waits for refresh. Execution remains bounded -by the Engine process capacity limits. +can execute while an earlier batch waits for refresh. New arrivals remain subject to process memory admission. Completed document chains release their queued successors even if the same execution still contains other unfinished documents. Conditional/Lua failures from speculative state are not final until the chain commits or definitively @@ -340,7 +339,7 @@ fails. A shared backend bulk still has to return before its results are known. The Engine's automatic batching across RPCs is always active. Explicit batches, operation folding in the core, adapter bulk operations, Kafka consumption -batches, core admission checks, and backend concurrency limits also apply. +batches, process memory admission, and backend concurrency limits also apply. Ordering guarantees have a scope. Within one request, operations for the same full address follow input order. With stable partition routing, Kafka @@ -389,7 +388,7 @@ Use these entry points to keep this guide aligned with future changes: | SDK batch splitting and Write RPCs | [sink-go client.go at the reviewed revision](https://github.com/liran/sink-go/blob/a658b054cea20c71f753ce21a5754885fc254318/client.go) | | Gateway, Engine, and Worker component wiring | `New` in [app.go](../internal/app/app.go) | | Synchronous batching and completion-mode separation | [batching_server.go](../internal/service/batching_server.go), [mutation_batches.go](../internal/service/mutation_batches.go) | -| Request dispatch, admission control, and Put/Merge execution | `Write` in [server.go](../internal/service/server.go), [admission.go](../internal/service/admission.go), [write.go](../internal/service/write.go) | +| Request dispatch, admission control, and Put/Merge execution | `Write` in [server.go](../internal/service/server.go), [memory guard](../internal/capacity/guard.go), [write.go](../internal/service/write.go) | | Write folding for one document | [write_group.go](../internal/service/write_group.go), [folding contract](merge-folding.md) | | Publishing original intent to Kafka | [publish.go](../internal/service/publish.go), [publisher.go](../internal/queue/kafka/publisher.go) | | Consumption, retries, DLQ, and offset commits | [worker.go](../internal/queue/kafka/worker.go), [processor.go](../internal/worker/processor.go) | From fbb92e02554815e2a866f3b946fc521cac718dc3 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 07:09:26 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=94=A7=20ci:=20pin=20qualification=20?= =?UTF-8?q?with=20valid=20startup=20memory=20fixtures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adf483a..94d00ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@5b364be341f1bc0bca6050f67b6d20893ee760f0 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@c97a8a6c601c9e1b62bccc74cbc979a7fea1c92a with: - suite_ref: 5b364be341f1bc0bca6050f67b6d20893ee760f0 + suite_ref: c97a8a6c601c9e1b62bccc74cbc979a7fea1c92a sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} From fb5859c0cbe4181f94306f97082a6499349751ef Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 07:43:21 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=92=9A=20fix:=20pin=20deterministic?= =?UTF-8?q?=20concurrent=20history=20qualification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94d00ca..a8b68cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@c97a8a6c601c9e1b62bccc74cbc979a7fea1c92a + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@924aa80684708a06685625b8161fc3529c218529 with: - suite_ref: c97a8a6c601c9e1b62bccc74cbc979a7fea1c92a + suite_ref: 924aa80684708a06685625b8161fc3529c218529 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }}