Skip to content
This repository was archived by the owner on Nov 8, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion docs/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,18 @@ For default warn-level diagnostic logs and optional OTLP export, see the
Gateway has its own forwarding metrics and process readiness. Engine keeps the
existing execution metrics; Worker keeps its Kafka metrics. See the
[role-specific scaling and health contract](store-isolation.md#readiness-metrics-and-scaling).
Gateway request volume is independent of response status. Its error counter
includes only Gateway-originated failures; Engine RPC failures are recorded by
Engine and are not attributed to Gateway when forwarded to clients. The
`sink_gateway_requests_total` label change removes the former `code` label.
Engine supplies a bounded private trailer fingerprinting its exact error status.
Gateway excludes an error only when that trailer matches the received status,
including its details. Local dial/send/receive failures and response-size limits
that fail the public RPC remain visible in Gateway error metrics.
This marker does not change the public status, error details, or mutation
acceptance/retry semantics. During mixed-version
rollouts, errors from older Engines without the marker are conservatively counted
by Gateway until those Engines are upgraded.

Every role serves `/livez` and `/readyz` on its dedicated health listener,
configured by `health.address` (default `:8081`). Prometheus is independent: set
Expand All @@ -28,7 +40,9 @@ Sink metrics:
| Metric | Type | Labels | Meaning |
| --- | --- | --- | --- |
| `sink_build_info` | gauge | `version` | Build identity for the running Sink binary. |
| `sink_grpc_server_requests_total` | counter | `store`, `method`, `code` | Completed Sink gRPC requests by method and canonical gRPC status code. |
| `sink_gateway_requests_total` | counter | `method` | Completed public RPCs regardless of downstream outcome. |
| `sink_gateway_errors_total` | counter | `method`, `code` | Non-OK public gRPC statuses originating in Gateway, excluding Engine RPC failures. |
| `sink_grpc_server_requests_total` | counter | `store`, `method`, `code` | Completed Sink requests owned by this process; Engine records its private Forward status under the public method and canonical gRPC status code. |
| `sink_grpc_server_request_duration_seconds` | histogram | `store`, `method` | End-to-end Sink gRPC request latency. |
| `sink_grpc_server_operation_results_total` | counter | `store`, `method`, `status` | Per-operation batch results and native Execute `succeeded`/`failed` outcomes, including database errors delivered over gRPC OK. |
| `sink_batcher_batches_total` | counter | `store`, `method`, `reason` | Synchronous batches dispatched by flush reason. |
Expand Down
7 changes: 5 additions & 2 deletions docs/store-isolation.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,11 @@ failure does not disable Kafka acceptance, and Kafka failure does not disable
synchronous execution. Worker readiness checks its own dependencies. Gateway does
not proxy named dependency health services; its default gRPC health is process health.

Gateway exports bounded method/code labels with `sink_gateway_requests_total`,
`sink_gateway_request_duration_seconds`, `sink_gateway_engine_duration_seconds`,
Gateway exports request volume with `sink_gateway_requests_total{method}` and
its own non-OK RPC statuses with `sink_gateway_errors_total{method,code}`;
confirmed Engine errors are excluded from the Gateway error counter.
Other Gateway metrics include `sink_gateway_request_duration_seconds`,
`sink_gateway_engine_duration_seconds`,
`sink_gateway_in_flight_requests`, `sink_gateway_routes`,
and `sink_gateway_config_info`.
Engine retains the existing `sink_grpc_server_*`, in-flight request, batching and Lua
Expand Down
4 changes: 4 additions & 0 deletions internal/engine/backpressure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,9 @@ func TestEngineDistinguishesAdmissionFromUnknownNativeOutcome(t *testing.T) {
if err != failure || (len(marker) == 1) != (failure == backpressure.ErrBusy) || stream.response != nil {
t.Fatalf("unknown outcome or admission evidence changed: error=%v marker=%v", err, marker)
}
origin := stream.trailer.Get(forwarding.ErrorStatusTrailer)
if len(origin) != 1 || origin[0] != forwarding.ErrorStatusMarker(failure) {
t.Fatal("Engine error omitted exact ownership evidence")
}
}
}
3 changes: 3 additions & 0 deletions internal/engine/memory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ func TestEnginePressureRejectsBeforeExecution(t *testing.T) {
if marker := stream.trailer.Get(forwarding.NotStartedTrailer); len(marker) != 1 || marker[0] != "true" {
t.Fatal("rejection omitted not-started evidence")
}
if marker := stream.trailer.Get(forwarding.ErrorStatusTrailer); len(marker) != 1 || marker[0] != forwarding.ErrorStatusMarker(err) {
t.Fatal("memory rejection omitted error ownership evidence")
}
}

type forwardRecorder struct {
Expand Down
7 changes: 6 additions & 1 deletion internal/engine/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,12 @@ func New(opts Options) (*Server, error) {
func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ForwardResponse]) (err error) {
ctx := stream.Context()
started := time.Now()
defer func() { s.metrics.ObserveForward(req, status.Code(err), time.Since(started)) }()
defer func() {
s.metrics.ObserveForward(req, status.Code(err), time.Since(started))
if marker := forwarding.ErrorStatusMarker(err); marker != "" {
stream.SetTrailer(metadata.Pairs(forwarding.ErrorStatusTrailer, marker))
}
}()
defer func() {
// Direct Native admission can reject after validation, but still before
// execution. Do not turn that known rejection into an unknown mutation.
Expand Down
31 changes: 31 additions & 0 deletions internal/forwarding/protocol.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
// Package forwarding defines the private Gateway-to-Engine transport contract.
package forwarding

import (
"crypto/sha256"
"encoding/hex"

"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)

const Version = 8

// EnvelopeBytes leaves room for the private wrapper around a public message.
Expand All @@ -9,3 +17,26 @@ const EnvelopeBytes = 4096
// NotStartedTrailer is set only when Engine proves no backend work was started.
// Its absence leaves undelivered mutation outcomes unknown.
const NotStartedTrailer = "sink-forward-not-started"

// ErrorStatusTrailer fingerprints the exact error returned by the Engine handler.
// It is independent of execution/acceptance evidence in NotStartedTrailer.
const ErrorStatusTrailer = "sink-forward-error"

// ErrorStatusMarker uses the same context-error normalization as the gRPC server.
// A fixed-size digest avoids duplicating potentially large status details.
// An empty marker cannot establish ownership and must not suppress metrics.
func ErrorStatusMarker(err error) string {
if err == nil {
return ""
}
reply, ok := status.FromError(err)
if !ok {
reply = status.FromContextError(err)
}
encoded, encodeErr := proto.Marshal(reply.Proto())
if encodeErr != nil {
return ""
}
digest := sha256.Sum256(encoded)
return hex.EncodeToString(digest[:])
}
44 changes: 44 additions & 0 deletions internal/forwarding/protocol_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package forwarding

import (
"context"
"errors"
"fmt"
"strings"
"testing"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

func TestErrorStatusMarkerNormalizesGRPCContextErrors(t *testing.T) {
failures := []error{
context.Canceled,
context.DeadlineExceeded,
fmt.Errorf("wrapped: %w", context.Canceled),
errors.New("plain handler failure"),
}
for _, failure := range failures {
t.Run(failure.Error(), func(t *testing.T) {
wireError := status.FromContextError(failure).Err()
marker := ErrorStatusMarker(failure)
if len(marker) != 64 || marker != ErrorStatusMarker(wireError) {
t.Fatalf("marker does not match gRPC wire status for %v", failure)
}
})
}
}

func TestErrorStatusMarkerIsBoundedAndRequiresEncodableError(t *testing.T) {
if marker := ErrorStatusMarker(nil); marker != "" {
t.Fatal("successful calls must not claim an error")
}
failure := status.Error(codes.ResourceExhausted, strings.Repeat("diagnostic", 1024))
if marker := ErrorStatusMarker(failure); len(marker) != 64 {
t.Fatalf("marker length=%d, want bounded SHA-256 hex", len(marker))
}
invalid := status.Error(codes.Internal, string([]byte{0xff}))
if marker := ErrorStatusMarker(invalid); marker != "" {
t.Fatal("unencodable error must not establish ownership")
}
}
212 changes: 212 additions & 0 deletions internal/gateway/forwarding_metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
package gateway

import (
"context"
"io"
"testing"

sink "github.com/batchstream/sink-protocol/sink/v1"
forward "github.com/batchstream/sink/gen/forward"
"github.com/batchstream/sink/internal/config"
"github.com/batchstream/sink/internal/forwarding"
"google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)

type metricsEngineClient struct {
stream *metricsEngineStream
failure error
}

func (c *metricsEngineClient) Forward(context.Context, *forward.ForwardRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[forward.ForwardResponse], error) {
return c.stream, c.failure
}

type metricsEngineStream struct {
grpc.ClientStream
frames []*forward.ForwardResponse
failure error
trailers metadata.MD
}

func (s *metricsEngineStream) Recv() (*forward.ForwardResponse, error) {
if len(s.frames) > 0 {
frame := s.frames[0]
s.frames = s.frames[1:]
return frame, nil
}
if s.failure != nil {
return nil, s.failure
}
return nil, io.EOF
}

func (s *metricsEngineStream) Trailer() metadata.MD { return s.trailers }

type metricsQueryStream struct {
grpc.ServerStream
ctx context.Context
}

func (s *metricsQueryStream) Context() context.Context { return s.ctx }
func (s *metricsQueryStream) Send(*sink.QueryResponse) error { return nil }

func TestForwardingMetricsRequireConfirmedEngineError(t *testing.T) {
detail := &errdetails.ErrorInfo{Domain: "sink", Reason: "ENGINE_CAPACITY_REJECTED"}
rejection, err := status.New(codes.ResourceExhausted, "capacity exhausted").WithDetails(detail)
if err != nil {
t.Fatal(err)
}
failure := rejection.Err()
marker := forwarding.ErrorStatusMarker(failure)
unavailable := status.Error(codes.Unavailable, "connection refused")
sendLimit := status.Error(codes.ResourceExhausted, "trying to send message larger than max")
receiveLimit := status.Error(codes.ResourceExhausted, "received message larger than max")
withoutDetails := status.Error(codes.ResourceExhausted, "capacity exhausted")
cases := []struct {
name string
startErr error
receiveErr error
markers []string
wantError error
gatewayErrors float64
}{
{name: "success"},
{name: "confirmed Engine rejection", receiveErr: failure, markers: []string{marker}, wantError: failure},
{name: "confirmed Engine unavailable", receiveErr: unavailable, markers: []string{forwarding.ErrorStatusMarker(unavailable)}, wantError: unavailable},
{name: "Gateway RPC initiation failed", startErr: unavailable, wantError: unavailable, gatewayErrors: 1},
{name: "Gateway send limit", startErr: sendLimit, wantError: sendLimit, gatewayErrors: 1},
{name: "Gateway dial failed during receive", receiveErr: unavailable, wantError: unavailable, gatewayErrors: 1},
{name: "Gateway receive limit", receiveErr: receiveLimit, wantError: receiveLimit, gatewayErrors: 1},
{name: "receive failure with different Engine status", receiveErr: receiveLimit, markers: []string{marker}, wantError: receiveLimit, gatewayErrors: 1},
{name: "different status details", receiveErr: withoutDetails, markers: []string{marker}, wantError: withoutDetails, gatewayErrors: 1},
{name: "legacy Engine without marker", receiveErr: failure, wantError: failure, gatewayErrors: 1},
{name: "empty marker", receiveErr: failure, markers: []string{""}, wantError: failure, gatewayErrors: 1},
{name: "duplicate marker", receiveErr: failure, markers: []string{marker, marker}, wantError: failure, gatewayErrors: 1},
{name: "malformed marker", receiveErr: failure, markers: []string{"invalid"}, wantError: failure, gatewayErrors: 1},
}
for _, method := range []string{"Execute", "Query"} {
for _, test := range cases {
t.Run(method+"/"+test.name, func(t *testing.T) {
trailers := metadata.MD{forwarding.ErrorStatusTrailer: test.markers}
stream := &metricsEngineStream{failure: test.receiveErr, trailers: trailers}
if test.wantError == nil {
stream.frames = metricsTestFrames(method)
}
client := &metricsEngineClient{stream: stream, failure: test.startErr}
gateway := metricsTestGateway(client)
err := metricsTestCall(t, gateway, method)
if !proto.Equal(status.Convert(err).Proto(), status.Convert(test.wantError).Proto()) {
t.Fatalf("public status or details changed: got %v, want %v", err, test.wantError)
}
requests, failures := metricsTestCounts(t, gateway)
var totalFailures float64
for _, count := range failures {
totalFailures += count
}
code := status.Code(test.wantError).String()
if requests != 1 || totalFailures != test.gatewayErrors || failures[code] != test.gatewayErrors {
t.Fatalf("Gateway requests=%v failures=%v, want requests=1 own errors=%v", requests, failures, test.gatewayErrors)
}
})
}
}
}

func TestForwardingErrorOwnershipDoesNotImplyNonExecution(t *testing.T) {
failure := status.Error(codes.ResourceExhausted, "Engine failure")
for _, notStarted := range []bool{false, true} {
trailers := metadata.Pairs(forwarding.ErrorStatusTrailer, forwarding.ErrorStatusMarker(failure))
if notStarted {
trailers.Set(forwarding.NotStartedTrailer, "true")
}
stream := &metricsEngineStream{failure: failure, trailers: trailers}
client := &metricsEngineClient{stream: stream}
gateway := metricsTestGateway(client)
command := &sink.Command{Uri: "sink://primary/fixture"}
execute := &sink.ExecuteRequest{Command: command}
body := &forward.ForwardRequest_Execute{Execute: execute}
request := &forward.ForwardRequest{Request: body}
call := forwardCall{route: gateway.current.routes["primary"], request: request}
actual, err := gateway.forwardEach(t.Context(), call)
if actual != notStarted || status.Code(err) != codes.ResourceExhausted {
t.Fatalf("non-execution evidence changed: notStarted=%t want=%t error=%v", actual, notStarted, err)
}
}
}

func metricsTestGateway(client forward.EngineClient) *Server {
route := Route{Store: "primary", Target: "passthrough:///metrics-test", endpoint: "metrics-test"}
entry := &connection{client: client}
current := &snapshot{routes: map[string]Route{"primary": route}}
request := config.Request{MaxOperations: 10, MaxReadBytes: 1 << 20}
gateway := &Server{metrics: newMetrics(), current: current, request: request}
gateway.pool.entries = map[Route]*connection{route: entry}
gateway.pool.maximum = 1
return gateway
}

func metricsTestFrames(method string) []*forward.ForwardResponse {
if method == "Execute" {
reply := &sink.ExecuteResponse{Success: true}
body := &forward.ForwardResponse_Execute{Execute: reply}
frame := &forward.ForwardResponse{Version: forwarding.Version, Store: "primary", Response: body}
frames := []*forward.ForwardResponse{frame}
return frames
}
reply := &sink.QueryResponse{Complete: true}
body := &forward.ForwardResponse_Query{Query: reply}
frame := &forward.ForwardResponse{Version: forwarding.Version, Store: "primary", Response: body}
frames := []*forward.ForwardResponse{frame}
return frames
}

func metricsTestCall(t *testing.T, gateway *Server, method string) error {
t.Helper()
command := &sink.Command{Uri: "sink://primary/fixture"}
if method == "Execute" {
request := &sink.ExecuteRequest{Command: command}
info := &grpc.UnaryServerInfo{FullMethod: sink.Sink_Execute_FullMethodName}
handler := func(ctx context.Context, _ any) (any, error) {
return gateway.Execute(ctx, request)
}
_, err := gateway.UnaryInterceptor()(t.Context(), request, info, handler)
return err
}
request := &sink.QueryRequest{Command: command}
info := &grpc.StreamServerInfo{FullMethod: sink.Sink_Query_FullMethodName, IsServerStream: true}
stream := &metricsQueryStream{ctx: t.Context()}
handler := func(any, grpc.ServerStream) error {
return gateway.Query(request, stream)
}
return gateway.StreamInterceptor()(gateway, stream, info, handler)
}

func metricsTestCounts(t *testing.T, gateway *Server) (float64, map[string]float64) {
t.Helper()
families, err := gateway.metrics.registry.Gather()
if err != nil {
t.Fatal(err)
}
var requests float64
failures := make(map[string]float64)
for _, family := range families {
for _, metric := range family.GetMetric() {
switch family.GetName() {
case "sink_gateway_requests_total":
requests += metric.GetCounter().GetValue()
case "sink_gateway_errors_total":
for _, label := range metric.GetLabel() {
if label.GetName() == "code" {
failures[label.GetValue()] += metric.GetCounter().GetValue()
}
}
}
}
}
return requests, failures
}
Loading
Loading