diff --git a/docs/observability.md b/docs/observability.md index 3313be7..888419c 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -6,6 +6,18 @@ For default warn-level diagnostic logs and optional OTLP export, see the Gateway has its own forwarding metrics and process readiness. Engine keeps the existing execution metrics; Worker keeps its Kafka metrics. See the [role-specific scaling and health contract](store-isolation.md#readiness-metrics-and-scaling). +Gateway request volume is independent of response status. Its error counter +includes only Gateway-originated failures; Engine RPC failures are recorded by +Engine and are not attributed to Gateway when forwarded to clients. The +`sink_gateway_requests_total` label change removes the former `code` label. +Engine supplies a bounded private trailer fingerprinting its exact error status. +Gateway excludes an error only when that trailer matches the received status, +including its details. Local dial/send/receive failures and response-size limits +that fail the public RPC remain visible in Gateway error metrics. +This marker does not change the public status, error details, or mutation +acceptance/retry semantics. During mixed-version +rollouts, errors from older Engines without the marker are conservatively counted +by Gateway until those Engines are upgraded. Every role serves `/livez` and `/readyz` on its dedicated health listener, configured by `health.address` (default `:8081`). Prometheus is independent: set @@ -28,7 +40,9 @@ Sink metrics: | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | | `sink_build_info` | gauge | `version` | Build identity for the running Sink binary. | -| `sink_grpc_server_requests_total` | counter | `store`, `method`, `code` | Completed Sink gRPC requests by method and canonical gRPC status code. | +| `sink_gateway_requests_total` | counter | `method` | Completed public RPCs regardless of downstream outcome. | +| `sink_gateway_errors_total` | counter | `method`, `code` | Non-OK public gRPC statuses originating in Gateway, excluding Engine RPC failures. | +| `sink_grpc_server_requests_total` | counter | `store`, `method`, `code` | Completed Sink requests owned by this process; Engine records its private Forward status under the public method and canonical gRPC status code. | | `sink_grpc_server_request_duration_seconds` | histogram | `store`, `method` | End-to-end Sink gRPC request latency. | | `sink_grpc_server_operation_results_total` | counter | `store`, `method`, `status` | Per-operation batch results and native Execute `succeeded`/`failed` outcomes, including database errors delivered over gRPC OK. | | `sink_batcher_batches_total` | counter | `store`, `method`, `reason` | Synchronous batches dispatched by flush reason. | diff --git a/docs/store-isolation.md b/docs/store-isolation.md index b782581..be77dd5 100644 --- a/docs/store-isolation.md +++ b/docs/store-isolation.md @@ -127,8 +127,11 @@ failure does not disable Kafka acceptance, and Kafka failure does not disable synchronous execution. Worker readiness checks its own dependencies. Gateway does not proxy named dependency health services; its default gRPC health is process health. -Gateway exports bounded method/code labels with `sink_gateway_requests_total`, -`sink_gateway_request_duration_seconds`, `sink_gateway_engine_duration_seconds`, +Gateway exports request volume with `sink_gateway_requests_total{method}` and +its own non-OK RPC statuses with `sink_gateway_errors_total{method,code}`; +confirmed Engine errors are excluded from the Gateway error counter. +Other Gateway metrics include `sink_gateway_request_duration_seconds`, +`sink_gateway_engine_duration_seconds`, `sink_gateway_in_flight_requests`, `sink_gateway_routes`, and `sink_gateway_config_info`. Engine retains the existing `sink_grpc_server_*`, in-flight request, batching and Lua diff --git a/internal/engine/backpressure_test.go b/internal/engine/backpressure_test.go index 39e4711..45efd88 100644 --- a/internal/engine/backpressure_test.go +++ b/internal/engine/backpressure_test.go @@ -39,5 +39,9 @@ func TestEngineDistinguishesAdmissionFromUnknownNativeOutcome(t *testing.T) { if err != failure || (len(marker) == 1) != (failure == backpressure.ErrBusy) || stream.response != nil { t.Fatalf("unknown outcome or admission evidence changed: error=%v marker=%v", err, marker) } + origin := stream.trailer.Get(forwarding.ErrorStatusTrailer) + if len(origin) != 1 || origin[0] != forwarding.ErrorStatusMarker(failure) { + t.Fatal("Engine error omitted exact ownership evidence") + } } } diff --git a/internal/engine/memory_test.go b/internal/engine/memory_test.go index 6dd5171..76e03b8 100644 --- a/internal/engine/memory_test.go +++ b/internal/engine/memory_test.go @@ -52,6 +52,9 @@ func TestEnginePressureRejectsBeforeExecution(t *testing.T) { if marker := stream.trailer.Get(forwarding.NotStartedTrailer); len(marker) != 1 || marker[0] != "true" { t.Fatal("rejection omitted not-started evidence") } + if marker := stream.trailer.Get(forwarding.ErrorStatusTrailer); len(marker) != 1 || marker[0] != forwarding.ErrorStatusMarker(err) { + t.Fatal("memory rejection omitted error ownership evidence") + } } type forwardRecorder struct { diff --git a/internal/engine/server.go b/internal/engine/server.go index bc25bed..5ea5b9a 100644 --- a/internal/engine/server.go +++ b/internal/engine/server.go @@ -49,7 +49,12 @@ func New(opts Options) (*Server, error) { func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ForwardResponse]) (err error) { ctx := stream.Context() started := time.Now() - defer func() { s.metrics.ObserveForward(req, status.Code(err), time.Since(started)) }() + defer func() { + s.metrics.ObserveForward(req, status.Code(err), time.Since(started)) + if marker := forwarding.ErrorStatusMarker(err); marker != "" { + stream.SetTrailer(metadata.Pairs(forwarding.ErrorStatusTrailer, marker)) + } + }() defer func() { // Direct Native admission can reject after validation, but still before // execution. Do not turn that known rejection into an unknown mutation. diff --git a/internal/forwarding/protocol.go b/internal/forwarding/protocol.go index bb73463..f58e01f 100644 --- a/internal/forwarding/protocol.go +++ b/internal/forwarding/protocol.go @@ -1,6 +1,14 @@ // Package forwarding defines the private Gateway-to-Engine transport contract. package forwarding +import ( + "crypto/sha256" + "encoding/hex" + + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + const Version = 8 // EnvelopeBytes leaves room for the private wrapper around a public message. @@ -9,3 +17,26 @@ const EnvelopeBytes = 4096 // NotStartedTrailer is set only when Engine proves no backend work was started. // Its absence leaves undelivered mutation outcomes unknown. const NotStartedTrailer = "sink-forward-not-started" + +// ErrorStatusTrailer fingerprints the exact error returned by the Engine handler. +// It is independent of execution/acceptance evidence in NotStartedTrailer. +const ErrorStatusTrailer = "sink-forward-error" + +// ErrorStatusMarker uses the same context-error normalization as the gRPC server. +// A fixed-size digest avoids duplicating potentially large status details. +// An empty marker cannot establish ownership and must not suppress metrics. +func ErrorStatusMarker(err error) string { + if err == nil { + return "" + } + reply, ok := status.FromError(err) + if !ok { + reply = status.FromContextError(err) + } + encoded, encodeErr := proto.Marshal(reply.Proto()) + if encodeErr != nil { + return "" + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/forwarding/protocol_test.go b/internal/forwarding/protocol_test.go new file mode 100644 index 0000000..ebae358 --- /dev/null +++ b/internal/forwarding/protocol_test.go @@ -0,0 +1,44 @@ +package forwarding + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestErrorStatusMarkerNormalizesGRPCContextErrors(t *testing.T) { + failures := []error{ + context.Canceled, + context.DeadlineExceeded, + fmt.Errorf("wrapped: %w", context.Canceled), + errors.New("plain handler failure"), + } + for _, failure := range failures { + t.Run(failure.Error(), func(t *testing.T) { + wireError := status.FromContextError(failure).Err() + marker := ErrorStatusMarker(failure) + if len(marker) != 64 || marker != ErrorStatusMarker(wireError) { + t.Fatalf("marker does not match gRPC wire status for %v", failure) + } + }) + } +} + +func TestErrorStatusMarkerIsBoundedAndRequiresEncodableError(t *testing.T) { + if marker := ErrorStatusMarker(nil); marker != "" { + t.Fatal("successful calls must not claim an error") + } + failure := status.Error(codes.ResourceExhausted, strings.Repeat("diagnostic", 1024)) + if marker := ErrorStatusMarker(failure); len(marker) != 64 { + t.Fatalf("marker length=%d, want bounded SHA-256 hex", len(marker)) + } + invalid := status.Error(codes.Internal, string([]byte{0xff})) + if marker := ErrorStatusMarker(invalid); marker != "" { + t.Fatal("unencodable error must not establish ownership") + } +} diff --git a/internal/gateway/forwarding_metrics_test.go b/internal/gateway/forwarding_metrics_test.go new file mode 100644 index 0000000..4e94b20 --- /dev/null +++ b/internal/gateway/forwarding_metrics_test.go @@ -0,0 +1,212 @@ +package gateway + +import ( + "context" + "io" + "testing" + + sink "github.com/batchstream/sink-protocol/sink/v1" + forward "github.com/batchstream/sink/gen/forward" + "github.com/batchstream/sink/internal/config" + "github.com/batchstream/sink/internal/forwarding" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type metricsEngineClient struct { + stream *metricsEngineStream + failure error +} + +func (c *metricsEngineClient) Forward(context.Context, *forward.ForwardRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[forward.ForwardResponse], error) { + return c.stream, c.failure +} + +type metricsEngineStream struct { + grpc.ClientStream + frames []*forward.ForwardResponse + failure error + trailers metadata.MD +} + +func (s *metricsEngineStream) Recv() (*forward.ForwardResponse, error) { + if len(s.frames) > 0 { + frame := s.frames[0] + s.frames = s.frames[1:] + return frame, nil + } + if s.failure != nil { + return nil, s.failure + } + return nil, io.EOF +} + +func (s *metricsEngineStream) Trailer() metadata.MD { return s.trailers } + +type metricsQueryStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s *metricsQueryStream) Context() context.Context { return s.ctx } +func (s *metricsQueryStream) Send(*sink.QueryResponse) error { return nil } + +func TestForwardingMetricsRequireConfirmedEngineError(t *testing.T) { + detail := &errdetails.ErrorInfo{Domain: "sink", Reason: "ENGINE_CAPACITY_REJECTED"} + rejection, err := status.New(codes.ResourceExhausted, "capacity exhausted").WithDetails(detail) + if err != nil { + t.Fatal(err) + } + failure := rejection.Err() + marker := forwarding.ErrorStatusMarker(failure) + unavailable := status.Error(codes.Unavailable, "connection refused") + sendLimit := status.Error(codes.ResourceExhausted, "trying to send message larger than max") + receiveLimit := status.Error(codes.ResourceExhausted, "received message larger than max") + withoutDetails := status.Error(codes.ResourceExhausted, "capacity exhausted") + cases := []struct { + name string + startErr error + receiveErr error + markers []string + wantError error + gatewayErrors float64 + }{ + {name: "success"}, + {name: "confirmed Engine rejection", receiveErr: failure, markers: []string{marker}, wantError: failure}, + {name: "confirmed Engine unavailable", receiveErr: unavailable, markers: []string{forwarding.ErrorStatusMarker(unavailable)}, wantError: unavailable}, + {name: "Gateway RPC initiation failed", startErr: unavailable, wantError: unavailable, gatewayErrors: 1}, + {name: "Gateway send limit", startErr: sendLimit, wantError: sendLimit, gatewayErrors: 1}, + {name: "Gateway dial failed during receive", receiveErr: unavailable, wantError: unavailable, gatewayErrors: 1}, + {name: "Gateway receive limit", receiveErr: receiveLimit, wantError: receiveLimit, gatewayErrors: 1}, + {name: "receive failure with different Engine status", receiveErr: receiveLimit, markers: []string{marker}, wantError: receiveLimit, gatewayErrors: 1}, + {name: "different status details", receiveErr: withoutDetails, markers: []string{marker}, wantError: withoutDetails, gatewayErrors: 1}, + {name: "legacy Engine without marker", receiveErr: failure, wantError: failure, gatewayErrors: 1}, + {name: "empty marker", receiveErr: failure, markers: []string{""}, wantError: failure, gatewayErrors: 1}, + {name: "duplicate marker", receiveErr: failure, markers: []string{marker, marker}, wantError: failure, gatewayErrors: 1}, + {name: "malformed marker", receiveErr: failure, markers: []string{"invalid"}, wantError: failure, gatewayErrors: 1}, + } + for _, method := range []string{"Execute", "Query"} { + for _, test := range cases { + t.Run(method+"/"+test.name, func(t *testing.T) { + trailers := metadata.MD{forwarding.ErrorStatusTrailer: test.markers} + stream := &metricsEngineStream{failure: test.receiveErr, trailers: trailers} + if test.wantError == nil { + stream.frames = metricsTestFrames(method) + } + client := &metricsEngineClient{stream: stream, failure: test.startErr} + gateway := metricsTestGateway(client) + err := metricsTestCall(t, gateway, method) + if !proto.Equal(status.Convert(err).Proto(), status.Convert(test.wantError).Proto()) { + t.Fatalf("public status or details changed: got %v, want %v", err, test.wantError) + } + requests, failures := metricsTestCounts(t, gateway) + var totalFailures float64 + for _, count := range failures { + totalFailures += count + } + code := status.Code(test.wantError).String() + if requests != 1 || totalFailures != test.gatewayErrors || failures[code] != test.gatewayErrors { + t.Fatalf("Gateway requests=%v failures=%v, want requests=1 own errors=%v", requests, failures, test.gatewayErrors) + } + }) + } + } +} + +func TestForwardingErrorOwnershipDoesNotImplyNonExecution(t *testing.T) { + failure := status.Error(codes.ResourceExhausted, "Engine failure") + for _, notStarted := range []bool{false, true} { + trailers := metadata.Pairs(forwarding.ErrorStatusTrailer, forwarding.ErrorStatusMarker(failure)) + if notStarted { + trailers.Set(forwarding.NotStartedTrailer, "true") + } + stream := &metricsEngineStream{failure: failure, trailers: trailers} + client := &metricsEngineClient{stream: stream} + gateway := metricsTestGateway(client) + command := &sink.Command{Uri: "sink://primary/fixture"} + execute := &sink.ExecuteRequest{Command: command} + body := &forward.ForwardRequest_Execute{Execute: execute} + request := &forward.ForwardRequest{Request: body} + call := forwardCall{route: gateway.current.routes["primary"], request: request} + actual, err := gateway.forwardEach(t.Context(), call) + if actual != notStarted || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("non-execution evidence changed: notStarted=%t want=%t error=%v", actual, notStarted, err) + } + } +} + +func metricsTestGateway(client forward.EngineClient) *Server { + route := Route{Store: "primary", Target: "passthrough:///metrics-test", endpoint: "metrics-test"} + entry := &connection{client: client} + current := &snapshot{routes: map[string]Route{"primary": route}} + request := config.Request{MaxOperations: 10, MaxReadBytes: 1 << 20} + gateway := &Server{metrics: newMetrics(), current: current, request: request} + gateway.pool.entries = map[Route]*connection{route: entry} + gateway.pool.maximum = 1 + return gateway +} + +func metricsTestFrames(method string) []*forward.ForwardResponse { + if method == "Execute" { + reply := &sink.ExecuteResponse{Success: true} + body := &forward.ForwardResponse_Execute{Execute: reply} + frame := &forward.ForwardResponse{Version: forwarding.Version, Store: "primary", Response: body} + frames := []*forward.ForwardResponse{frame} + return frames + } + reply := &sink.QueryResponse{Complete: true} + body := &forward.ForwardResponse_Query{Query: reply} + frame := &forward.ForwardResponse{Version: forwarding.Version, Store: "primary", Response: body} + frames := []*forward.ForwardResponse{frame} + return frames +} + +func metricsTestCall(t *testing.T, gateway *Server, method string) error { + t.Helper() + command := &sink.Command{Uri: "sink://primary/fixture"} + if method == "Execute" { + request := &sink.ExecuteRequest{Command: command} + info := &grpc.UnaryServerInfo{FullMethod: sink.Sink_Execute_FullMethodName} + handler := func(ctx context.Context, _ any) (any, error) { + return gateway.Execute(ctx, request) + } + _, err := gateway.UnaryInterceptor()(t.Context(), request, info, handler) + return err + } + request := &sink.QueryRequest{Command: command} + info := &grpc.StreamServerInfo{FullMethod: sink.Sink_Query_FullMethodName, IsServerStream: true} + stream := &metricsQueryStream{ctx: t.Context()} + handler := func(any, grpc.ServerStream) error { + return gateway.Query(request, stream) + } + return gateway.StreamInterceptor()(gateway, stream, info, handler) +} + +func metricsTestCounts(t *testing.T, gateway *Server) (float64, map[string]float64) { + t.Helper() + families, err := gateway.metrics.registry.Gather() + if err != nil { + t.Fatal(err) + } + var requests float64 + failures := make(map[string]float64) + for _, family := range families { + for _, metric := range family.GetMetric() { + switch family.GetName() { + case "sink_gateway_requests_total": + requests += metric.GetCounter().GetValue() + case "sink_gateway_errors_total": + for _, label := range metric.GetLabel() { + if label.GetName() == "code" { + failures[label.GetValue()] += metric.GetCounter().GetValue() + } + } + } + } + } + return requests, failures +} diff --git a/internal/gateway/metrics.go b/internal/gateway/metrics.go index 6c2cbec..f3d52d2 100644 --- a/internal/gateway/metrics.go +++ b/internal/gateway/metrics.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "net/http" "strings" "time" @@ -17,6 +18,7 @@ type metrics struct { registry *prometheus.Registry config *prometheus.GaugeVec requests *prometheus.CounterVec + failures *prometheus.CounterVec duration *prometheus.HistogramVec inFlight prometheus.Gauge routes prometheus.Gauge @@ -31,14 +33,25 @@ func newMetrics() *metrics { observed := &metrics{registry: registry, inFlight: prometheus.NewGauge(inFlightOpts), routes: prometheus.NewGauge(routesOpts), downstream: prometheus.NewHistogram(downstreamOpts)} configOpts := prometheus.GaugeOpts{Name: "sink_gateway_config_info", Help: "SHA-256 of the normalized routes loaded at startup."} observed.config = prometheus.NewGaugeVec(configOpts, []string{"sha256"}) - requestOpts := prometheus.CounterOpts{Name: "sink_gateway_requests_total", Help: "Completed public RPCs."} - observed.requests = prometheus.NewCounterVec(requestOpts, []string{"method", "code"}) + requestOpts := prometheus.CounterOpts{Name: "sink_gateway_requests_total", Help: "Completed public RPCs, regardless of downstream outcome."} + observed.requests = prometheus.NewCounterVec(requestOpts, []string{"method"}) + failureOpts := prometheus.CounterOpts{Name: "sink_gateway_errors_total", Help: "Public RPC failures originating in Gateway, excluding Engine RPC failures."} + observed.failures = prometheus.NewCounterVec(failureOpts, []string{"method", "code"}) durationOpts := prometheus.HistogramOpts{Name: "sink_gateway_request_duration_seconds", Help: "Public RPC latency.", Buckets: prometheus.DefBuckets} observed.duration = prometheus.NewHistogramVec(durationOpts, []string{"method"}) processOpts := collectors.ProcessCollectorOpts{} - registry.MustRegister(observed.config, observed.requests, observed.duration, observed.inFlight, observed.routes, observed.downstream, collectors.NewGoCollector(), collectors.NewProcessCollector(processOpts)) + registry.MustRegister(observed.config, observed.requests, observed.failures, observed.duration, observed.inFlight, observed.routes, observed.downstream, collectors.NewGoCollector(), collectors.NewProcessCollector(processOpts)) return observed } + +func (m *metrics) observeRequest(method string, err error) { + m.requests.WithLabelValues(method).Inc() + var downstream *downstreamError + if err != nil && !errors.As(err, &downstream) { + m.failures.WithLabelValues(method, status.Code(err).String()).Inc() + } +} + func (s *Server) MetricsHandler() http.Handler { opts := promhttp.HandlerOpts{} return promhttp.HandlerFor(s.metrics.registry, opts) @@ -51,7 +64,7 @@ func (s *Server) UnaryInterceptor() grpc.UnaryServerInterceptor { method := strings.TrimPrefix(info.FullMethod, "/sink.v1.Sink/") switch method { case "Read", "Write", "Delete", "Execute", "Query", "Count", "Scan": - s.metrics.requests.WithLabelValues(method, status.Code(err).String()).Inc() + s.metrics.observeRequest(method, err) s.metrics.duration.WithLabelValues(method).Observe(time.Since(started).Seconds()) } return response, err @@ -65,7 +78,7 @@ func (s *Server) StreamInterceptor() grpc.StreamServerInterceptor { method := strings.TrimPrefix(info.FullMethod, "/sink.v1.Sink/") switch method { case "Read", "Write", "Query", "Scan": - s.metrics.requests.WithLabelValues(method, status.Code(err).String()).Inc() + s.metrics.observeRequest(method, err) s.metrics.duration.WithLabelValues(method).Observe(time.Since(started).Seconds()) } return err diff --git a/internal/gateway/metrics_test.go b/internal/gateway/metrics_test.go new file mode 100644 index 0000000..22a3f5d --- /dev/null +++ b/internal/gateway/metrics_test.go @@ -0,0 +1,50 @@ +package gateway + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestGatewayMetricsExcludeDownstreamFailures(t *testing.T) { + server := &Server{metrics: newMetrics()} + interceptor := server.UnaryInterceptor() + info := &grpc.UnaryServerInfo{FullMethod: "/sink.v1.Sink/Count"} + downstreamErr := status.Error(codes.ResourceExhausted, "Engine admission rejected request") + markedDownstreamErr := &downstreamError{cause: downstreamErr} + downstreamHandler := func(context.Context, any) (any, error) { + return nil, markedDownstreamErr + } + _, err := interceptor(context.Background(), nil, info, downstreamHandler) + if status.Code(err) != codes.ResourceExhausted { + t.Fatalf("downstream status = %s, want ResourceExhausted", status.Code(err)) + } + + gatewayErr := status.Error(codes.ResourceExhausted, "Gateway capacity exhausted") + gatewayHandler := func(context.Context, any) (any, error) { + return nil, gatewayErr + } + _, err = interceptor(context.Background(), nil, info, gatewayHandler) + if status.Code(err) != codes.ResourceExhausted { + t.Fatalf("Gateway status = %s, want ResourceExhausted", status.Code(err)) + } + + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/metrics", nil) + server.MetricsHandler().ServeHTTP(recorder, request) + body := recorder.Body.String() + if !strings.Contains(body, `sink_gateway_requests_total{method="Count"} 2`) { + t.Fatalf("request counter missing: %s", body) + } + if !strings.Contains(body, `sink_gateway_errors_total{code="ResourceExhausted",method="Count"} 1`) { + t.Fatalf("Gateway-owned error counter missing: %s", body) + } + if strings.Contains(body, `sink_gateway_errors_total{code="ResourceExhausted",method="Count"} 2`) { + t.Fatalf("downstream failure was attributed to Gateway: %s", body) + } +} diff --git a/internal/gateway/stream.go b/internal/gateway/stream.go index 0e81085..4962d7f 100644 --- a/internal/gateway/stream.go +++ b/internal/gateway/stream.go @@ -17,6 +17,13 @@ type forwardCall struct { emit func(*forward.ForwardResponse) error } +// downstreamError preserves the Engine RPC status without attributing it to Gateway. +type downstreamError struct{ cause error } + +func (e *downstreamError) Error() string { return e.cause.Error() } +func (e *downstreamError) Unwrap() error { return e.cause } +func (e *downstreamError) GRPCStatus() *status.Status { return status.Convert(e.cause) } + // forwardEach reads the next result only after the public send completes. The // transport therefore supplies backpressure without an intermediate queue. func (s *Server) forwardEach(ctx context.Context, call forwardCall) (bool, error) { @@ -55,6 +62,11 @@ func (s *Server) forwardEach(ctx context.Context, call forwardCall) (bool, error if err == io.EOF { return false, nil } + engineStatus := stream.Trailer().Get(forwarding.ErrorStatusTrailer) + if len(engineStatus) == 1 && engineStatus[0] != "" && engineStatus[0] == forwarding.ErrorStatusMarker(err) { + downstream := &downstreamError{cause: err} + return notStarted, downstream + } return notStarted, err } if frame.GetVersion() != forwarding.Version || frame.GetStore() != route.Store {