From 28ffa0e2765375dc64a6fdadef48443da0fad8e8 Mon Sep 17 00:00:00 2001 From: Juho Makinen Date: Wed, 4 Nov 2026 14:35:12 +1100 Subject: [PATCH 1/5] feat(server)!: stream everything a mirror sends the server over one stream A mirror's state and its changes now travel over one ordered stream, so the server learns of a quarantine as it happens from the same place as everything else, and later traffic such as captures can share it instead of opening its own connection. The reconnecting, resyncing and keepalives are generic, so the next kind of message only adds a variant. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 17 +- cli_internal_test.go | 21 +- demo/README.md | 2 +- demo/entrypoint.sh | 2 +- internal/cli/commandline.go | 2 +- internal/cli/status_internal_test.go | 3 +- internal/model/state.go | 18 +- internal/model/update.go | 20 ++ internal/server/client/client.go | 40 ++- internal/server/config.go | 6 +- internal/server/reporter/reporter.go | 98 ++++--- internal/server/reporter/reporter_test.go | 51 ++-- internal/server/server_test.go | 146 +++++++--- internal/server/service.go | 82 +++++- internal/server/storage/storage_test.go | 4 +- internal/server/wire/wire.go | 68 ++++- internal/stream/handler.go | 44 +++ internal/stream/outbox.go | 153 +++++++++++ internal/stream/outbox_test.go | 165 +++++++++++ internal/supervisor/supervisor.go | 2 +- proto/spectre/v1/spectre.proto | 18 +- spectrepb/spectre.pb.go | 256 +++++++++++++----- spectrepb/spectrepbconnect/spectre.connect.go | 75 +++-- 23 files changed, 1014 insertions(+), 279 deletions(-) create mode 100644 internal/model/update.go create mode 100644 internal/stream/handler.go create mode 100644 internal/stream/outbox.go create mode 100644 internal/stream/outbox_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7667be9..cb14a7a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,9 +64,8 @@ and `docker.io/blockossreleases/spectre`. ## SPECTRE server -`spectre serve` collects the heartbeats that mirrors send. Each heartbeat -carries the mirror's latest state: its candidate revision, SPECTRE version, -uptime, and quarantines. The server also records each quarantine as a +`spectre serve` collects the states that mirrors stream to it: each mirror's +candidate revision, SPECTRE version, uptime, and quarantines. The server also records each quarantine as a divergence that outlives the mirror run. It listens on `127.0.0.1:50070` by default and keeps everything in memory, so a restart loses it. To keep them in Redis instead, pass a Redis URL to `--storage`, such as `--storage=redis://localhost:6379/0`. For a Redis Cluster, use the @@ -86,14 +85,18 @@ pass the socket as `--server-socket` and the server's mesh address as `--server-url`. Every call dials the socket, and the mesh routes it by the URL's host, such as `--server-url=http://spectre.mesh --server-socket=@egress.sock`. -A mirror sends a heartbeat when it starts, every `--heartbeat-interval` (30s by -default), and as soon as it quarantines. A failed heartbeat is only logged. +A mirror sends everything to the server over one stream, which begins with its +whole state. It then sends each quarantine as it begins, and a keepalive every +`--server-keepalive-interval` (30s by default) when nothing else is sent. When +the stream fails, the mirror logs it and reconnects with backoff, from 1s up to +1m. It also replaces the stream after `--server-stream-max-age` (10m by +default, with jitter), so streams spread across server replicas. List the reporting mirrors and recorded divergences with `spectre status --server-url=URL`. Filter with `--service`, `--environment`, `--revision`, or `--node`, and print JSON with `--json`. A -mirror stays listed until it misses heartbeats for the server's `--state-ttl` -(90s by default), so keep that above the heartbeat interval. A mirror stays +mirror stays listed until it sends nothing for the server's `--state-ttl` +(90s by default), so keep that above the keepalive interval. A mirror stays quarantined until it restarts. The server keeps the latest divergence of each direction of a service diff --git a/cli_internal_test.go b/cli_internal_test.go index 490039e..b11c4b5 100644 --- a/cli_internal_test.go +++ b/cli_internal_test.go @@ -17,6 +17,7 @@ import ( "github.com/block/spectre/internal/server/client" "github.com/block/spectre/internal/server/reporter" "github.com/block/spectre/internal/server/storage" + "github.com/block/spectre/internal/stream" "github.com/block/spectre/internal/supervisor" ) @@ -28,11 +29,11 @@ func reportingFlags() []string { // expectedReporting is the reporter config that reportingFlags parse to. func expectedReporting() reporter.Config { return reporter.Config{ - Client: client.Config{URL: "http://server.example"}, - Service: "users", - Environment: "production", - Revision: "abc123", - HeartbeatInterval: 30 * time.Second, + Client: client.Config{URL: "http://server.example"}, + Service: "users", + Environment: "production", + Revision: "abc123", + Stream: stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: 10 * time.Minute}, } } @@ -85,11 +86,11 @@ func TestSuperviseCommandReadsFlagsFromEnvironment(t *testing.T) { }) assert.NoError(t, err) expected := reporter.Config{ - Client: client.Config{URL: "http://server.example"}, - Service: "invoices", - Environment: "production-us-west-2", - Revision: "3f9c2a1", - HeartbeatInterval: 30 * time.Second, + Client: client.Config{URL: "http://server.example"}, + Service: "invoices", + Environment: "production-us-west-2", + Revision: "3f9c2a1", + Stream: stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: 10 * time.Minute}, } assert.Equal(t, expected, command.Supervise.Supervisor.Server) } diff --git a/demo/README.md b/demo/README.md index f870dd1..5f7c61e 100644 --- a/demo/README.md +++ b/demo/README.md @@ -62,7 +62,7 @@ Base your image on the same pattern as `demo/Dockerfile`: (`--scripts-dir`), TypeScript schema (`--schema-dir`), and protobuf descriptors (`--descriptors-dir`). 3. Run the supervisor with your two commands, pointed at a SPECTRE server - (`spectre serve`) that it sends heartbeats to. + (`spectre serve`) that it streams its state to. The supervisor runs each service command through `/bin/sh -c` and exports two environment variables to it: diff --git a/demo/entrypoint.sh b/demo/entrypoint.sh index f8fef3f..1d510db 100755 --- a/demo/entrypoint.sh +++ b/demo/entrypoint.sh @@ -13,7 +13,7 @@ spectre-sample-forecasts \ --listen="unix:$sockets/forecasts.sock" \ --weather=/spectre/testdata/weather.json & -# Mirrors send heartbeats to the SPECTRE server, so run one alongside. +# Mirrors stream their states to the SPECTRE server, so run one alongside. spectre serve --listen=127.0.0.1:50070 & service="spectre-sample --listen=\$SPECTRE_LISTEN --forecasts=\$SPECTRE_EGRESS --data=/spectre/testdata/users.json --weather=/spectre/testdata/weather.json" diff --git a/internal/cli/commandline.go b/internal/cli/commandline.go index 1fba8cb..cc5760f 100644 --- a/internal/cli/commandline.go +++ b/internal/cli/commandline.go @@ -27,7 +27,7 @@ type CommandLine struct { Version kong.VersionFlag `env:"-" help:"Print the version and exit."` Supervise superviseCommand `cmd:"" help:"Supervise a candidate and a reference behind the embedded proxies, splitting resources between them."` - Serve serveCommand `cmd:"" help:"Run the SPECTRE server, which mirrors send their heartbeats to."` + Serve serveCommand `cmd:"" help:"Run the SPECTRE server, which mirrors stream their states to."` Gen genCommand `cmd:"" help:"Generate TypeScript schema declarations and the script API."` Status statusCommand `cmd:"" help:"List the mirrors reporting to the SPECTRE server and the divergences they reported, or clear divergences."` State stateCommand `cmd:"" help:"Describe a service's migration state."` diff --git a/internal/cli/status_internal_test.go b/internal/cli/status_internal_test.go index 44b0ddf..6bed0eb 100644 --- a/internal/cli/status_internal_test.go +++ b/internal/cli/status_internal_test.go @@ -54,8 +54,9 @@ func TestStatusCommands(t *testing.T) { Uptime: 12*time.Minute + 300*time.Millisecond, } for _, state := range []model.State{quarantined, healthy} { - assert.NoError(t, spectre.Update(t.Context(), state)) + assert.NoError(t, store.Put(t.Context(), state, time.Now())) } + assert.NoError(t, store.Record(t.Context(), model.Divergence{Service: "users", Environment: "production", Revision: "def456", Direction: model.Ingress, Quarantine: quarantined.Quarantines[model.Ingress]})) var text strings.Builder list := statusListCommand{} diff --git a/internal/model/state.go b/internal/model/state.go index bce22a3..9704ce8 100644 --- a/internal/model/state.go +++ b/internal/model/state.go @@ -1,9 +1,12 @@ package model -import "time" +import ( + "maps" + "time" +) // State is what the mirror of a service environment runs and holds, as its -// latest heartbeat reported it. +// latest update reported it. type State struct { Service string `json:"service"` Environment string `json:"environment"` @@ -16,3 +19,14 @@ type State struct { // Quarantines holds the quarantine of each direction whose proxy has one. Quarantines map[Direction]Quarantine `json:"quarantines,omitempty"` } + +func (State) update() {} + +// Quarantined returns the state with the quarantine of the proxy in direction. +func (s State) Quarantined(direction Direction, quarantine Quarantine) State { + quarantines := make(map[Direction]Quarantine, len(s.Quarantines)+1) + maps.Copy(quarantines, s.Quarantines) + quarantines[direction] = quarantine + s.Quarantines = quarantines + return s +} diff --git a/internal/model/update.go b/internal/model/update.go new file mode 100644 index 0000000..4d705f0 --- /dev/null +++ b/internal/model/update.go @@ -0,0 +1,20 @@ +package model + +// Update is one message of the stream a mirror sends the SPECTRE server. Every +// stream begins with the mirror's whole State. +// +//sumtype:decl +type Update interface{ update() } + +// QuarantineUpdate adds the quarantine of the proxy in Direction to the state. +type QuarantineUpdate struct { + Direction Direction + Quarantine Quarantine +} + +func (QuarantineUpdate) update() {} + +// Keepalive shows the server that a quiet mirror still runs. +type Keepalive struct{} + +func (Keepalive) update() {} diff --git a/internal/server/client/client.go b/internal/server/client/client.go index 656f042..55ed08f 100644 --- a/internal/server/client/client.go +++ b/internal/server/client/client.go @@ -5,7 +5,6 @@ import ( "context" "net" "net/http" - "time" "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" @@ -17,9 +16,6 @@ import ( "github.com/block/spectre/spectrepb/spectrepbconnect" ) -// heartbeatTimeout bounds one heartbeat, so a slow server never delays the next. -const heartbeatTimeout = 10 * time.Second - // Config locates the SPECTRE server. type Config struct { URL string `name:"server-url" default:"${server_url}" placeholder:"URL" help:"Base URL of the SPECTRE server."` @@ -56,13 +52,14 @@ func New(config Config) *Client { return &Client{rpc: spectrepbconnect.NewSpectreServiceClient(connect.NewClient(connecthttp.NewTransport(httpClient, config.URL)))} } -// Update sends a mirror's latest state. It does not retry, because the -// next heartbeat replaces it. -func (c *Client) Update(ctx context.Context, state model.State) error { - ctx, cancel := context.WithTimeout(ctx, heartbeatTimeout) - defer cancel() - _, err := c.rpc.Update(ctx, &pb.UpdateRequest{State: wire.StateMessage(state)}) - return errors.Wrap(err, "send heartbeat") +// OpenUpdates opens a stream of a mirror's updates, which lasts until ctx ends +// or the stream is closed. +func (c *Client) OpenUpdates(ctx context.Context) (*Updates, error) { + stream, err := c.rpc.Update(ctx) + if err != nil { + return nil, errors.Wrap(err, "open update stream") + } + return newUpdates(stream), nil } // ListDivergences returns the states of the reporting mirrors and the @@ -82,3 +79,24 @@ func (c *Client) ClearDivergences(ctx context.Context, service, environment stri _, err := c.rpc.ClearDivergences(ctx, request) return errors.Wrap(err, "clear divergences") } + +// Updates streams changes to a mirror's state to the SPECTRE server. +type Updates struct { + stream spectrepbconnect.SpectreServiceUpdateClientStream +} + +func newUpdates(stream spectrepbconnect.SpectreServiceUpdateClientStream) *Updates { + return &Updates{stream: stream} +} + +// Send sends update. It fails once the server has ended the stream, and +// CloseAndReceive then returns why it did. +func (u *Updates) Send(update model.Update) error { + return errors.Wrap(u.stream.Send(wire.UpdateMessage(update)), "send update") +} + +// CloseAndReceive ends the stream and returns the error the server ended it with. +func (u *Updates) CloseAndReceive() error { + _, err := u.stream.CloseAndReceive() + return errors.Wrap(err, "close update stream") +} diff --git a/internal/server/config.go b/internal/server/config.go index 517b150..2e4537e 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -1,5 +1,5 @@ -// Package server is the SPECTRE server, which collects the heartbeats that -// mirrors send. It keeps their states and divergences in memory or in Redis. +// Package server is the SPECTRE server, which collects the states that mirrors +// stream. It keeps their states and divergences in memory or in Redis. package server import ( @@ -12,7 +12,7 @@ import ( type ServeConfig struct { Listen string `default:"127.0.0.1:50070" help:"Address for the SPECTRE server: host:port, or unix:."` Storage storage.Config `embed:""` - StateTTL time.Duration `default:"90s" help:"How long a mirror's state is listed after its last heartbeat. Keep it above the heartbeat interval."` + StateTTL time.Duration `default:"90s" help:"How long a mirror's state is listed after its last update. Keep it above the mirrors' keepalive interval."` } // ClearConfig names the service environment whose divergences to clear. diff --git a/internal/server/reporter/reporter.go b/internal/server/reporter/reporter.go index 9c9486b..e2d364c 100644 --- a/internal/server/reporter/reporter.go +++ b/internal/server/reporter/reporter.go @@ -1,24 +1,27 @@ -// Package reporter sends a mirror's heartbeats to the SPECTRE server. +// Package reporter streams what a mirror sends the SPECTRE server: its state +// and every change to it, over one stream. package reporter import ( "context" "maps" + "sync" "time" "github.com/block/spectre/internal" "github.com/block/spectre/internal/logger" "github.com/block/spectre/internal/model" "github.com/block/spectre/internal/server/client" + "github.com/block/spectre/internal/stream" ) -// Config configures how a mirror reports its state to the SPECTRE server. +// Config configures how a mirror reports to the SPECTRE server. type Config struct { - Client client.Config `embed:""` - Service string `required:"" help:"Service the mirror runs, reported to the SPECTRE server."` - Environment string `required:"" help:"Environment the mirror runs in, such as production-us-west-2, reported to the SPECTRE server."` - Revision string `required:"" help:"Commit of the candidate the mirror runs, reported to the SPECTRE server."` - HeartbeatInterval time.Duration `default:"30s" help:"How often the mirror sends its state to the SPECTRE server."` + Client client.Config `embed:""` + Stream stream.Config `embed:"" prefix:"server-"` + Service string `required:"" help:"Service the mirror runs, reported to the SPECTRE server."` + Environment string `required:"" help:"Environment the mirror runs in, such as production-us-west-2, reported to the SPECTRE server."` + Revision string `required:"" help:"Commit of the candidate the mirror runs, reported to the SPECTRE server."` } // Source is a proxy whose quarantine a mirror reports. @@ -28,71 +31,62 @@ type Source struct { Quarantined <-chan model.Quarantine } -// Reporter sends a mirror's state to the SPECTRE server. +// Reporter streams a mirror's state to the SPECTRE server. type Reporter struct { config Config client *client.Client sources []Source + started time.Time + + mu sync.Mutex + quarantines map[model.Direction]model.Quarantine } // New reports the state of a mirror and its sources through server. func New(config Config, server *client.Client, sources ...Source) *Reporter { - return &Reporter{config: config, client: server, sources: sources} -} - -// sourceQuarantine is a quarantine of the source in direction. -type sourceQuarantine struct { - direction model.Direction - quarantine model.Quarantine + return &Reporter{config: config, client: server, sources: sources, started: time.Now(), quarantines: map[model.Direction]model.Quarantine{}} } -// Run sends a heartbeat at once, at every interval, and when a source -// quarantines, until ctx is cancelled. +// Run streams the mirror's state until ctx is cancelled. Every stream begins +// with the whole state, then sends each quarantine as it begins. func (r *Reporter) Run(ctx context.Context) { - started := time.Now() - quarantined := make(chan sourceQuarantine) + dial := func(ctx context.Context) (stream.Conn[model.Update], error) { + return r.client.OpenUpdates(ctx) + } + resync := func() []model.Update { return []model.Update{r.state()} } + keepalive := func() model.Update { return model.Keepalive{} } + outbox := stream.NewOutbox(r.config.Stream, dial, resync, keepalive) for _, source := range r.sources { - go forward(ctx, source, quarantined) + go r.report(ctx, source, outbox) } - quarantines := map[model.Direction]model.Quarantine{} - ticker := time.NewTicker(r.config.HeartbeatInterval) - defer ticker.Stop() - for { - r.heartbeat(ctx, time.Since(started), quarantines) - select { - case <-ctx.Done(): - return - case <-ticker.C: - case status := <-quarantined: - quarantines[status.direction] = status.quarantine + outbox.Run(ctx) +} + +// report sends the source's quarantine once it begins. The state holds it too, +// so a stream that fails before sending it resyncs it. +func (r *Reporter) report(ctx context.Context, source Source, outbox *stream.Outbox[model.Update]) { + select { + case quarantine := <-source.Quarantined: + r.mu.Lock() + r.quarantines[source.Direction] = quarantine + r.mu.Unlock() + if !outbox.Send(model.QuarantineUpdate{Direction: source.Direction, Quarantine: quarantine}) { + logger.FromContext(ctx).WarnContext(ctx, "Dropped a quarantine update for the SPECTRE server; the next stream resyncs it") } + case <-ctx.Done(): } } -// heartbeat sends the mirror's state. A failure is only logged, so the -// SPECTRE server never blocks the mirror. -func (r *Reporter) heartbeat(ctx context.Context, uptime time.Duration, quarantines map[model.Direction]model.Quarantine) { - state := model.State{ +// state returns the mirror's whole state. +func (r *Reporter) state() model.State { + r.mu.Lock() + defer r.mu.Unlock() + return model.State{ Service: r.config.Service, Environment: r.config.Environment, Revision: r.config.Revision, SpectreVersion: internal.Version, - Uptime: uptime, - Quarantines: maps.Clone(quarantines), - } - if err := r.client.Update(ctx, state); err != nil && ctx.Err() == nil { - logger.FromContext(ctx).WarnContext(ctx, "Failed to send heartbeat to the SPECTRE server", "error", err) - } -} - -// forward passes the source's quarantine on to quarantined. -func forward(ctx context.Context, source Source, quarantined chan<- sourceQuarantine) { - select { - case status := <-source.Quarantined: - select { - case quarantined <- sourceQuarantine{direction: source.Direction, quarantine: status}: - case <-ctx.Done(): - } - case <-ctx.Done(): + Uptime: time.Since(r.started), + Quarantines: maps.Clone(r.quarantines), } } diff --git a/internal/server/reporter/reporter_test.go b/internal/server/reporter/reporter_test.go index 6d7b620..f74ad8c 100644 --- a/internal/server/reporter/reporter_test.go +++ b/internal/server/reporter/reporter_test.go @@ -17,53 +17,62 @@ import ( "github.com/block/spectre/internal/server/client" "github.com/block/spectre/internal/server/reporter" "github.com/block/spectre/internal/server/storage" + "github.com/block/spectre/internal/stream" ) -// updateProcedure is the protocol's path for sending a heartbeat. +// updateProcedure is the protocol's path for streaming a mirror's state. const updateProcedure = "/spectre.v1.SpectreService/Update" -func TestReporterSendsHeartbeatOnQuarantine(t *testing.T) { - heartbeats := make(chan int, 8) +func TestReporterReconnectsWithQuarantine(t *testing.T) { var attempts atomic.Int32 service := server.NewService(storage.NewMemory(), time.Hour).Handler() httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - if request.URL.Path != updateProcedure { - service.ServeHTTP(w, request) - return - } - attempt := int(attempts.Add(1)) - // The first heartbeat fails, which the next one recovers from. - if attempt == 1 { + // The first stream fails, which the reporter recovers from. + if request.URL.Path == updateProcedure && attempts.Add(1) == 1 { + // Reply without reading the stream, as a proxy in front of the server does. + _ = http.NewResponseController(w).SetReadDeadline(time.Now()) //nolint:errcheck // HTTP/1 test server http.Error(w, "unavailable", http.StatusServiceUnavailable) - } else { - service.ServeHTTP(w, request) + return } - heartbeats <- attempt + service.ServeHTTP(w, request) })) t.Cleanup(httpServer.Close) ingress := make(chan model.Quarantine, 1) egress := make(chan model.Quarantine, 1) config := reporter.Config{ - Client: client.Config{URL: httpServer.URL}, - Service: "users", - Environment: "production", - Revision: "abc123", - HeartbeatInterval: time.Hour, + Client: client.Config{URL: httpServer.URL}, + Service: "users", + Environment: "production", + Revision: "abc123", + Stream: stream.Config{KeepaliveInterval: 10 * time.Millisecond, MaxAge: time.Hour}, } spectre := client.New(config.Client) mirror := reporter.New(config, spectre, reporter.Source{Direction: model.Ingress, Quarantined: ingress}, reporter.Source{Direction: model.Egress, Quarantined: egress}) ctx, cancel := context.WithCancel(t.Context()) t.Cleanup(cancel) go mirror.Run(ctx) - assert.Equal(t, 1, <-heartbeats) - quarantine := model.Quarantine{Since: time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC), Reason: model.QuarantineReason{ Kind: model.QuarantineUnmatched, Traffic: Some(model.Traffic{Direction: model.Egress, Protocol: model.ProtocolHTTP, Route: "POST api.example/v1/charges"}), Detail: "no matching reference call", }} egress <- quarantine - assert.Equal(t, 2, <-heartbeats) + + // Whichever stream the quarantine reaches, the next one begins with it. + deadline := time.After(10 * time.Second) + for { + _, divergences, err := spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) + assert.NoError(t, err) + if len(divergences) > 0 { + break + } + select { + case <-deadline: + t.Fatal("the reporter never reported the quarantine") + case <-time.After(10 * time.Millisecond): + } + } + assert.True(t, attempts.Load() > 1) states, divergences, err := spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) assert.NoError(t, err) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 6fafd44..5f30288 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,6 +1,7 @@ package server_test import ( + "io" "net/http/httptest" "testing" "time" @@ -8,6 +9,7 @@ import ( "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" "github.com/alecthomas/assert/v2" + "github.com/alecthomas/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -36,6 +38,54 @@ func state(environment string, uptime time.Duration, quarantines ...*pb.Quaranti } } +// update streams requests to the server, then closes the stream and returns +// the error the server ended it with. +func update(t *testing.T, rpc spectrepbconnect.SpectreServiceClient, requests ...*pb.UpdateRequest) error { + t.Helper() + stream, err := rpc.Update(t.Context()) + assert.NoError(t, err) + for _, request := range requests { + // A send fails with io.EOF once the server has ended the stream. + if err := stream.Send(request); err != nil && !errors.Is(err, io.EOF) { + return err + } + } + _, err = stream.CloseAndReceive() + return err +} + +func whole(state *pb.State) *pb.UpdateRequest { + return &pb.UpdateRequest{Update: &pb.UpdateRequest_State{State: state}} +} + +func quarantined(quarantine *pb.Quarantine) *pb.UpdateRequest { + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Quarantine{Quarantine: quarantine}} +} + +func keepalive() *pb.UpdateRequest { + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Keepalive{Keepalive: &pb.Keepalive{}}} +} + +// list returns what the server lists for request without uptimes, which it +// returns separately because the server advances them. +func list(t *testing.T, rpc spectrepbconnect.SpectreServiceClient, request *pb.ListDivergencesRequest) (listed *pb.ListDivergencesResponse, uptimes []time.Duration) { + t.Helper() + response, err := rpc.ListDivergences(t.Context(), request) + assert.NoError(t, err) + for _, listedState := range response.GetStates() { + uptimes = append(uptimes, listedState.GetUptime().AsDuration()) + listedState.Uptime = nil + } + return response, uptimes +} + +// withoutUptime returns a copy of state without its uptime, as list returns it. +func withoutUptime(state *pb.State) *pb.State { + stripped := proto.CloneOf(state) + stripped.Uptime = nil + return stripped +} + func TestUpdateKeepsLatestState(t *testing.T) { rpc := newRPC(t) since := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) @@ -47,67 +97,60 @@ func TestUpdateKeepsLatestState(t *testing.T) { Node: proto.String("users"), Differences: []string{"body.name"}, } - latest := state("production", 2*time.Minute, quarantine) - divergence := &pb.Divergence{Service: "users", Environment: "production", Revision: "abc123", Quarantine: quarantine} staging := state("staging", time.Minute) - for _, sent := range []*pb.State{state("production", time.Minute), staging, latest} { - _, err := rpc.Update(t.Context(), &pb.UpdateRequest{State: sent}) - assert.NoError(t, err) - } + assert.NoError(t, update(t, rpc, whole(state("production", time.Hour)), keepalive(), quarantined(quarantine), keepalive())) + assert.NoError(t, update(t, rpc, whole(staging))) - all := &pb.ListDivergencesResponse{States: []*pb.State{latest, staging}, Divergences: []*pb.Divergence{divergence}} + production := withoutUptime(state("production", time.Hour, quarantine)) + divergence := &pb.Divergence{Service: "users", Environment: "production", Revision: "abc123", Quarantine: quarantine} tests := []struct { name string request *pb.ListDivergencesRequest expected *pb.ListDivergencesResponse }{ - {name: "All", request: &pb.ListDivergencesRequest{}, expected: all}, - {name: "Environment", request: &pb.ListDivergencesRequest{Environment: proto.String("staging")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{staging}}}, - {name: "Node", request: &pb.ListDivergencesRequest{Node: proto.String("users")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{latest}, Divergences: []*pb.Divergence{divergence}}}, + {name: "All", request: &pb.ListDivergencesRequest{}, expected: &pb.ListDivergencesResponse{States: []*pb.State{production, withoutUptime(staging)}, Divergences: []*pb.Divergence{divergence}}}, + {name: "Environment", request: &pb.ListDivergencesRequest{Environment: proto.String("staging")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{withoutUptime(staging)}}}, + {name: "Node", request: &pb.ListDivergencesRequest{Node: proto.String("users")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{production}, Divergences: []*pb.Divergence{divergence}}}, {name: "Revision", request: &pb.ListDivergencesRequest{Revision: proto.String("def456")}, expected: &pb.ListDivergencesResponse{}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - listed, err := rpc.ListDivergences(t.Context(), test.request) - assert.NoError(t, err) + listed, _ := list(t, rpc, test.request) assert.True(t, proto.Equal(test.expected, listed), "%v", listed) }) } + + // Later updates advance the uptime the mirror sent with its whole state. + _, uptimes := list(t, rpc, &pb.ListDivergencesRequest{}) + assert.True(t, uptimes[0] > time.Hour && uptimes[0] < time.Hour+time.Minute, "%v", uptimes[0]) + assert.Equal(t, time.Minute, uptimes[1]) } func TestDivergencesOutliveRestartUntilCleared(t *testing.T) { rpc := newRPC(t) quarantine := &pb.Quarantine{Direction: pb.Direction_DIRECTION_EGRESS, Since: timestamppb.Now(), Kind: pb.Quarantine_KIND_UNMATCHED} - quarantined := state("production", time.Minute, quarantine) redeployed := state("production", time.Second) redeployed.Revision = "def456" - heartbeat := func(sent *pb.State) { - _, err := rpc.Update(t.Context(), &pb.UpdateRequest{State: sent}) - assert.NoError(t, err) - } - list := func() *pb.ListDivergencesResponse { - listed, err := rpc.ListDivergences(t.Context(), &pb.ListDivergencesRequest{}) - assert.NoError(t, err) - return listed - } - heartbeat(quarantined) - heartbeat(redeployed) + assert.NoError(t, update(t, rpc, whole(state("production", time.Minute)), quarantined(quarantine))) + assert.NoError(t, update(t, rpc, whole(redeployed))) divergence := &pb.Divergence{Service: "users", Environment: "production", Revision: "abc123", Quarantine: quarantine} - expected := &pb.ListDivergencesResponse{States: []*pb.State{redeployed}, Divergences: []*pb.Divergence{divergence}} - assert.True(t, proto.Equal(expected, list()), "%v", list()) + expected := &pb.ListDivergencesResponse{States: []*pb.State{withoutUptime(redeployed)}, Divergences: []*pb.Divergence{divergence}} + listed, _ := list(t, rpc, &pb.ListDivergencesRequest{}) + assert.True(t, proto.Equal(expected, listed), "%v", listed) _, err := rpc.ClearDivergences(t.Context(), &pb.ClearDivergencesRequest{Service: "users", Environment: "production"}) assert.NoError(t, err) - // A heartbeat that was in flight still carries the cleared quarantine. - heartbeat(quarantined) - heartbeat(redeployed) - assert.True(t, proto.Equal(&pb.ListDivergencesResponse{States: []*pb.State{redeployed}}, list()), "%v", list()) + // A mirror still running reports the cleared quarantine again. + assert.NoError(t, update(t, rpc, whole(state("production", time.Minute, quarantine)))) + assert.NoError(t, update(t, rpc, whole(redeployed))) + listed, _ = list(t, rpc, &pb.ListDivergencesRequest{}) + assert.True(t, proto.Equal(&pb.ListDivergencesResponse{States: []*pb.State{withoutUptime(redeployed)}}, listed), "%v", listed) _, err = rpc.ClearDivergences(t.Context(), &pb.ClearDivergencesRequest{Service: "users"}) assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) } -func TestUpdateRejectsInvalidState(t *testing.T) { +func TestUpdateRejectsInvalidStream(t *testing.T) { rpc := newRPC(t) quarantine := func(direction pb.Direction) *pb.Quarantine { return &pb.Quarantine{Direction: direction, Since: timestamppb.Now(), Kind: pb.Quarantine_KIND_UNMATCHED} @@ -121,21 +164,42 @@ func TestUpdateRejectsInvalidState(t *testing.T) { withoutTrafficDirection := state("production", time.Minute, quarantine(pb.Direction_DIRECTION_INGRESS)) withoutTrafficDirection.Quarantines[0].Traffic = &pb.Traffic{Protocol: "http"} tests := []struct { - name string - state *pb.State + name string + requests []*pb.UpdateRequest }{ - {name: "WithoutState"}, - {name: "WithoutRevision", state: withoutCommit}, - {name: "WithoutUptime", state: withoutUptime}, - {name: "UnspecifiedQuarantineDirection", state: state("production", time.Minute, quarantine(pb.Direction_DIRECTION_UNSPECIFIED))}, - {name: "RepeatedDirection", state: state("production", time.Minute, quarantine(pb.Direction_DIRECTION_INGRESS), quarantine(pb.Direction_DIRECTION_INGRESS))}, - {name: "UnspecifiedKind", state: withoutKind}, - {name: "UnspecifiedTrafficDirection", state: withoutTrafficDirection}, + {name: "Empty", requests: []*pb.UpdateRequest{{}}}, + {name: "WithoutState", requests: []*pb.UpdateRequest{whole(nil)}}, + {name: "BeginsWithKeepalive", requests: []*pb.UpdateRequest{keepalive()}}, + {name: "BeginsWithQuarantine", requests: []*pb.UpdateRequest{quarantined(quarantine(pb.Direction_DIRECTION_INGRESS))}}, + {name: "WithoutRevision", requests: []*pb.UpdateRequest{whole(withoutCommit)}}, + {name: "WithoutUptime", requests: []*pb.UpdateRequest{whole(withoutUptime)}}, + {name: "UnspecifiedQuarantineDirection", requests: []*pb.UpdateRequest{whole(state("production", time.Minute, quarantine(pb.Direction_DIRECTION_UNSPECIFIED)))}}, + {name: "RepeatedDirection", requests: []*pb.UpdateRequest{whole(state("production", time.Minute, quarantine(pb.Direction_DIRECTION_INGRESS), quarantine(pb.Direction_DIRECTION_INGRESS)))}}, + {name: "UnspecifiedKind", requests: []*pb.UpdateRequest{whole(withoutKind)}}, + {name: "UnspecifiedTrafficDirection", requests: []*pb.UpdateRequest{whole(withoutTrafficDirection)}}, + {name: "InvalidQuarantine", requests: []*pb.UpdateRequest{whole(state("production", time.Minute)), quarantined(quarantine(pb.Direction_DIRECTION_UNSPECIFIED))}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := rpc.Update(t.Context(), &pb.UpdateRequest{State: test.state}) + err := update(t, rpc, test.requests...) assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) }) } } + +func TestUpdateEndsRejectedStreamWhileMirrorSends(t *testing.T) { + rpc := newRPC(t) + stream, err := rpc.Update(t.Context()) + assert.NoError(t, err) + assert.NoError(t, stream.Send(whole(state("production", time.Minute)))) + assert.NoError(t, stream.Send(quarantined(&pb.Quarantine{Since: timestamppb.Now(), Kind: pb.Quarantine_KIND_UNMATCHED}))) + // A mirror only learns that the server ended its stream when a send fails. + deadline := time.Now().Add(10 * time.Second) + for err == nil && time.Now().Before(deadline) { + err = stream.Send(keepalive()) + time.Sleep(time.Millisecond) + } + assert.IsError(t, err, io.EOF) + _, err = stream.CloseAndReceive() + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) +} diff --git a/internal/server/service.go b/internal/server/service.go index 6e06fbb..dd4c7ce 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -2,6 +2,7 @@ package server import ( "context" + "io" "net" "net/http" "time" @@ -9,6 +10,7 @@ import ( "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" "github.com/alecthomas/errors" + . "github.com/alecthomas/types/optional" "github.com/block/spectre/internal/httpserver/middleware/health" "github.com/block/spectre/internal/httpserver/middleware/logging" @@ -17,6 +19,7 @@ import ( "github.com/block/spectre/internal/netaddr" "github.com/block/spectre/internal/server/storage" "github.com/block/spectre/internal/server/wire" + "github.com/block/spectre/internal/stream" pb "github.com/block/spectre/spectrepb" "github.com/block/spectre/spectrepb/spectrepbconnect" ) @@ -39,19 +42,48 @@ func (s *Service) Handler() http.Handler { server := connect.NewServer() spectrepbconnect.RegisterSpectreServiceHandler(server, s) connecthttp.Mount(mux, server) - return mux + return stream.EndEarly(mux) } -// Update records the latest state of a mirror, and the divergence of each -// of its quarantines. -func (s *Service) Update(ctx context.Context, request *pb.UpdateRequest) (*pb.UpdateResponse, error) { - state, err := wire.StateFromMessage(request.GetState()) - if err != nil { - return nil, connect.NewError(connect.CodeInvalidArgument, err.Error()).WithCause(err) - } - if err := s.store.Put(ctx, state, time.Now()); err != nil { - return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) +// Update keeps the state a mirror streams. Every message stores the whole +// state, which keeps it listed, and records the divergence of each quarantine. +func (s *Service) Update(ctx context.Context, updates spectrepbconnect.SpectreServiceUpdateServerStream) (*pb.UpdateResponse, error) { + var state Option[model.State] + var measured time.Time + for { + message, err := updates.Receive() + if errors.Is(err, io.EOF) { + return &pb.UpdateResponse{}, nil + } + if err != nil { + return nil, errors.WithStack(err) + } + current, err := nextState(state, message) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err.Error()).WithCause(err) + } + now := time.Now() + if message.GetState() != nil { + measured = now + } + state = Some(current) + // The mirror measured its uptime when it sent the whole state. + stored := current + stored.Uptime += now.Sub(measured) + if err := s.store.Put(ctx, stored, now); err != nil { + return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) + } + if message.GetKeepalive() != nil { + continue + } + if err := s.recordQuarantines(ctx, current); err != nil { + return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) + } } +} + +// recordQuarantines records the divergence of each quarantine state holds. +func (s *Service) recordQuarantines(ctx context.Context, state model.State) error { for direction, quarantine := range state.Quarantines { divergence := model.Divergence{ Service: state.Service, @@ -61,14 +93,14 @@ func (s *Service) Update(ctx context.Context, request *pb.UpdateRequest) (*pb.Up Quarantine: quarantine, } if err := s.store.Record(ctx, divergence); err != nil { - return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) + return errors.WithStack(err) } } - return &pb.UpdateResponse{}, nil + return nil } -// ListDivergences returns the states of the mirrors that sent a heartbeat -// within the TTL, and every recorded divergence. +// ListDivergences returns the states of the mirrors that sent an update within +// the TTL, and every recorded divergence. func (s *Service) ListDivergences(ctx context.Context, request *pb.ListDivergencesRequest) (*pb.ListDivergencesResponse, error) { states, err := s.store.States(ctx, time.Now().Add(-s.ttl)) if err != nil { @@ -129,3 +161,25 @@ func Serve(ctx context.Context, config ServeConfig) error { } return nil } + +// nextState applies a streamed update to the state a stream has built. A +// stream must begin with the whole state. +func nextState(state Option[model.State], message *pb.UpdateRequest) (model.State, error) { + update, err := wire.UpdateFromMessage(message) + if err != nil { + return model.State{}, errors.WithStack(err) + } + if whole, ok := update.(model.State); ok { + return whole, nil + } + current, ok := state.Get() + if !ok { + return model.State{}, errors.New("an update stream must begin with the mirror's state") + } + switch update := update.(type) { + case model.QuarantineUpdate: + return current.Quarantined(update.Direction, update.Quarantine), nil + case model.State, model.Keepalive: + } + return current, nil +} diff --git a/internal/server/storage/storage_test.go b/internal/server/storage/storage_test.go index ec83fc8..f283256 100644 --- a/internal/server/storage/storage_test.go +++ b/internal/server/storage/storage_test.go @@ -252,10 +252,10 @@ func testKeepsLatestQuarantine(t *testing.T, store storage.Store) { for _, reported := range []model.Divergence{ divergence("production", model.Ingress, first), latest, - // A heartbeat from before the mirror restarted and quarantined again. + // An update from before the mirror restarted and quarantined again. divergence("production", model.Ingress, first.Add(-time.Minute)), egress, - // Every heartbeat repeats the quarantine. + // Every update repeats the quarantine. egress, } { assert.NoError(t, store.Record(t.Context(), reported)) diff --git a/internal/server/wire/wire.go b/internal/server/wire/wire.go index 99b0a76..0db875b 100644 --- a/internal/server/wire/wire.go +++ b/internal/server/wire/wire.go @@ -25,7 +25,7 @@ func StateMessage(state model.State) *pb.State { // Ingress first, so a state always encodes the same way. for _, direction := range []model.Direction{model.Ingress, model.Egress} { if quarantine, ok := state.Quarantines[direction]; ok { - message.Quarantines = append(message.Quarantines, quarantineMessage(direction, quarantine)) + message.Quarantines = append(message.Quarantines, QuarantineMessage(direction, quarantine)) } } return message @@ -52,29 +52,55 @@ func StateFromMessage(message *pb.State) (model.State, error) { state.Quarantines = make(map[model.Direction]model.Quarantine, len(quarantines)) } for _, quarantine := range quarantines { - direction, err := directionFromMessage(quarantine.GetDirection()) + direction, restored, err := QuarantineFromMessage(quarantine) if err != nil { - return model.State{}, errors.Wrap(err, "quarantine") + return model.State{}, err } if _, ok := state.Quarantines[direction]; ok { return model.State{}, errors.Errorf("state holds more than one %s quarantine", direction) } - restored, err := quarantineFromMessage(quarantine) - if err != nil { - return model.State{}, errors.Wrap(err, "quarantine") - } state.Quarantines[direction] = restored } return state, nil } +// UpdateMessage returns the message that streams update. +func UpdateMessage(update model.Update) *pb.UpdateRequest { + switch update := update.(type) { + case model.State: + return &pb.UpdateRequest{Update: &pb.UpdateRequest_State{State: StateMessage(update)}} + case model.QuarantineUpdate: + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Quarantine{Quarantine: QuarantineMessage(update.Direction, update.Quarantine)}} + case model.Keepalive: + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Keepalive{Keepalive: &pb.Keepalive{}}} + } + return &pb.UpdateRequest{} +} + +// UpdateFromMessage restores a streamed update. +func UpdateFromMessage(message *pb.UpdateRequest) (model.Update, error) { + switch update := message.GetUpdate().(type) { + case *pb.UpdateRequest_State: + return StateFromMessage(update.State) + case *pb.UpdateRequest_Quarantine: + direction, quarantine, err := QuarantineFromMessage(update.Quarantine) + if err != nil { + return nil, err + } + return model.QuarantineUpdate{Direction: direction, Quarantine: quarantine}, nil + case *pb.UpdateRequest_Keepalive: + return model.Keepalive{}, nil + } + return nil, errors.New("update holds nothing") +} + // DivergenceMessage returns the message that reports divergence. func DivergenceMessage(divergence model.Divergence) *pb.Divergence { return &pb.Divergence{ Service: divergence.Service, Environment: divergence.Environment, Revision: divergence.Revision, - Quarantine: quarantineMessage(divergence.Direction, divergence.Quarantine), + Quarantine: QuarantineMessage(divergence.Direction, divergence.Quarantine), } } @@ -87,13 +113,9 @@ func DivergenceFromMessage(message *pb.Divergence) (model.Divergence, error) { if message.GetQuarantine() == nil { return model.Divergence{}, errors.New("divergence needs a quarantine") } - direction, err := directionFromMessage(message.GetQuarantine().GetDirection()) - if err != nil { - return model.Divergence{}, errors.Wrap(err, "quarantine") - } - quarantine, err := quarantineFromMessage(message.GetQuarantine()) + direction, quarantine, err := QuarantineFromMessage(message.GetQuarantine()) if err != nil { - return model.Divergence{}, errors.Wrap(err, "quarantine") + return model.Divergence{}, err } return model.Divergence{ Service: message.GetService(), @@ -168,7 +190,9 @@ func FilterFromMessage(request *pb.ListDivergencesRequest) model.DivergenceFilte } } -func quarantineMessage(direction model.Direction, status model.Quarantine) *pb.Quarantine { +// QuarantineMessage returns the message that reports the quarantine of the +// proxy in direction. +func QuarantineMessage(direction model.Direction, status model.Quarantine) *pb.Quarantine { reason := status.Reason message := &pb.Quarantine{ Direction: directionMessage(direction), @@ -188,6 +212,20 @@ func quarantineMessage(direction model.Direction, status model.Quarantine) *pb.Q return message } +// QuarantineFromMessage restores a reported quarantine and the direction of +// the proxy that holds it. +func QuarantineFromMessage(message *pb.Quarantine) (model.Direction, model.Quarantine, error) { + direction, err := directionFromMessage(message.GetDirection()) + if err != nil { + return "", model.Quarantine{}, errors.Wrap(err, "quarantine") + } + quarantine, err := quarantineFromMessage(message) + if err != nil { + return "", model.Quarantine{}, errors.Wrap(err, "quarantine") + } + return direction, quarantine, nil +} + // quarantineFromMessage restores a reported quarantine, which must say when it began. func quarantineFromMessage(message *pb.Quarantine) (model.Quarantine, error) { if message.GetSince() == nil { diff --git a/internal/stream/handler.go b/internal/stream/handler.go new file mode 100644 index 0000000..85c2a36 --- /dev/null +++ b/internal/stream/handler.go @@ -0,0 +1,44 @@ +package stream + +import ( + "io" + "net/http" + "time" + + "github.com/alecthomas/errors" +) + +// EndEarly lets next end a client stream before the client does. Otherwise an +// HTTP/1 server reads the rest of the request before it replies. +func EndEarly(next http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + request.Body = newUnendedBody(request.Body, http.NewResponseController(writer)) + next.ServeHTTP(writer, request) + }) +} + +// unendedBody is a request body that the client may not have ended. +type unendedBody struct { + io.ReadCloser + controller *http.ResponseController + ended bool +} + +func newUnendedBody(body io.ReadCloser, controller *http.ResponseController) *unendedBody { + return &unendedBody{ReadCloser: body, controller: controller} +} + +func (b *unendedBody) Read(data []byte) (int, error) { + n, err := b.ReadCloser.Read(data) + b.ended = b.ended || errors.Is(err, io.EOF) + return n, err //nolint:wrapcheck // Readers must return io.EOF itself. +} + +// Close stops reading a body the client has not ended, so the server replies +// at once and then closes the connection, which tells the client. +func (b *unendedBody) Close() error { + if !b.ended { + _ = b.controller.SetReadDeadline(time.Now()) //nolint:errcheck // Without a deadline, closing only takes longer. + } + return errors.WithStack(b.ReadCloser.Close()) +} diff --git a/internal/stream/outbox.go b/internal/stream/outbox.go new file mode 100644 index 0000000..a0a5f5d --- /dev/null +++ b/internal/stream/outbox.go @@ -0,0 +1,153 @@ +// Package stream keeps long-lived client streams open. An Outbox keeps one +// stream from a client open through failures and sends it what producers queue. +// Every stream it opens begins with a resync of the client's whole state, so +// only events need queueing, and a stream may fail at any time without losing +// state. EndEarly lets a server end such a stream before the client does. +package stream + +import ( + "context" + "math/rand/v2" + "time" + + "github.com/alecthomas/errors" + + "github.com/block/spectre/internal/logger" +) + +// Conn is one open client stream. +type Conn[M any] interface { + // Send sends message. It fails once the server has ended the stream. + Send(message M) error + // CloseAndReceive ends the stream and returns the error the server ended it with. + CloseAndReceive() error +} + +// Config configures how an outbox keeps its stream open. +type Config struct { + KeepaliveInterval time.Duration `default:"30s" help:"How often to send a keepalive on a stream that is otherwise quiet."` + MaxAge time.Duration `name:"stream-max-age" default:"10m" help:"How long a stream lasts, with jitter, before it is replaced, so streams spread across servers."` +} + +// Validate checks that the intervals are positive. +func (c Config) Validate() error { + if c.KeepaliveInterval <= 0 || c.MaxAge <= 0 { + return errors.New("stream intervals must be positive") + } + return nil +} + +// Retries of a failed stream back off between these delays. +const ( + minRetryDelay = time.Second + maxRetryDelay = time.Minute +) + +// queueSize is how many messages wait for a stream before Send drops them. +const queueSize = 256 + +// Outbox keeps one stream open and sends it the messages producers queue. +type Outbox[M any] struct { + config Config + dial func(ctx context.Context) (Conn[M], error) + resync func() []M + keepalive func() M + queue chan M +} + +// NewOutbox returns an outbox that opens streams with dial. Each stream begins +// with the messages resync returns, and a quiet stream sends keepalive. +func NewOutbox[M any](config Config, dial func(ctx context.Context) (Conn[M], error), resync func() []M, keepalive func() M) *Outbox[M] { + return &Outbox[M]{config: config, dial: dial, resync: resync, keepalive: keepalive, queue: make(chan M, queueSize)} +} + +// Send queues message for the stream without blocking. It drops the message +// when the queue is full. A queued message is lost if its stream fails. +func (o *Outbox[M]) Send(message M) (queued bool) { + select { + case o.queue <- message: + return true + default: + return false + } +} + +// Run keeps a stream open until ctx ends, replacing it when it fails, with +// backoff, or when it reaches its age. Failures are only logged. +func (o *Outbox[M]) Run(ctx context.Context) { + delay := minRetryDelay + for { + opened := time.Now() + err := o.stream(ctx) + if ctx.Err() != nil { + return + } + // Only consecutive quick failures back off further. + if err == nil || time.Since(opened) > maxRetryDelay { + delay = minRetryDelay + } + if err == nil { + continue + } + wait := jitter(delay) + logger.FromContext(ctx).WarnContext(ctx, "Lost a stream to the server", "error", err, "retry", wait) + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + delay = min(delay*2, maxRetryDelay) + } +} + +// stream sends on one stream until it fails or ctx ends. It returns nil once +// the stream reaches its age and the server accepts it closing. +func (o *Outbox[M]) stream(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + conn, err := o.dial(ctx) + if err != nil { + return errors.Wrap(err, "open stream") + } + for _, message := range o.resync() { + if err := conn.Send(message); err != nil { + return failed(conn, err) + } + } + keepalive := time.NewTimer(o.config.KeepaliveInterval) + defer keepalive.Stop() + age := time.NewTimer(jitter(o.config.MaxAge)) + defer age.Stop() + for { + var message M + select { + case <-ctx.Done(): + cancel() + _ = conn.CloseAndReceive() //nolint:errcheck // Cancelling has already aborted the stream. + return errors.WithStack(ctx.Err()) + case <-age.C: + return errors.Wrap(conn.CloseAndReceive(), "close aged stream") + case <-keepalive.C: + message = o.keepalive() + case message = <-o.queue: + } + if err := conn.Send(message); err != nil { + return failed(conn, err) + } + keepalive.Reset(o.config.KeepaliveInterval) + } +} + +// failed closes a stream whose send failed. A send fails once the server has +// ended the stream, and closing returns why it did. +func failed[M any](conn Conn[M], err error) error { + if closeErr := conn.CloseAndReceive(); closeErr != nil { + return errors.Wrap(closeErr, "stream ended") + } + return errors.Wrap(err, "send on stream") +} + +// jitter returns a random duration between half of d and d. +func jitter(d time.Duration) time.Duration { + return d/2 + rand.N(d/2+1) +} diff --git a/internal/stream/outbox_test.go b/internal/stream/outbox_test.go new file mode 100644 index 0000000..b7d54a7 --- /dev/null +++ b/internal/stream/outbox_test.go @@ -0,0 +1,165 @@ +package stream_test + +import ( + "context" + "io" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/alecthomas/assert/v2" + "github.com/alecthomas/errors" + + "github.com/block/spectre/internal/stream" +) + +// conn records what an outbox sends on one stream. +type conn struct { + mu sync.Mutex + sent []string + // Sends fail with io.EOF once this many messages are sent, unless it is zero. + limit int + closed bool +} + +func (c *conn) Send(message string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.limit > 0 && len(c.sent) >= c.limit { + return io.EOF + } + c.sent = append(c.sent, message) + return nil +} + +func (c *conn) CloseAndReceive() error { + c.mu.Lock() + defer c.mu.Unlock() + c.closed = true + if c.limit > 0 { + return errors.New("server ended the stream") + } + return nil +} + +// sent is what a conn was sent and whether it was closed. +type sent struct { + Messages []string + Closed bool +} + +// dialer hands out a new conn for every stream an outbox opens. +type dialer struct { + mu sync.Mutex + conns []*conn + limit []int +} + +func (d *dialer) dial(context.Context) (stream.Conn[string], error) { + d.mu.Lock() + defer d.mu.Unlock() + opened := &conn{} + if len(d.conns) < len(d.limit) { + opened.limit = d.limit[len(d.conns)] + } + d.conns = append(d.conns, opened) + return opened, nil +} + +func (d *dialer) streams() []sent { + d.mu.Lock() + defer d.mu.Unlock() + streams := make([]sent, 0, len(d.conns)) + for _, opened := range d.conns { + opened.mu.Lock() + streams = append(streams, sent{Messages: append([]string(nil), opened.sent...), Closed: opened.closed}) + opened.mu.Unlock() + } + return streams +} + +// run runs an outbox until the test ends. +func run(t *testing.T, config stream.Config, d *dialer) *stream.Outbox[string] { + t.Helper() + outbox := stream.NewOutbox(config, d.dial, func() []string { return []string{"state"} }, func() string { return "keepalive" }) + ctx, cancel := context.WithCancel(t.Context()) + go outbox.Run(ctx) + t.Cleanup(func() { + cancel() + synctest.Wait() + }) + return outbox +} + +var config = stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: time.Hour} + +func TestOutboxResyncsThenSendsAndKeepsAlive(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &dialer{} + outbox := run(t, config, d) + assert.True(t, outbox.Send("event")) + synctest.Wait() + assert.Equal(t, []sent{{Messages: []string{"state", "event"}}}, d.streams()) + + // Only a quiet stream sends keepalives. + time.Sleep(20 * time.Second) + assert.True(t, outbox.Send("later")) + time.Sleep(20 * time.Second) + synctest.Wait() + assert.Equal(t, []sent{{Messages: []string{"state", "event", "later"}}}, d.streams()) + time.Sleep(10 * time.Second) + synctest.Wait() + assert.Equal(t, []sent{{Messages: []string{"state", "event", "later", "keepalive"}}}, d.streams()) + }) +} + +func TestOutboxReconnectsWithBackoff(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // The first two streams end after the resync, the third lasts. + d := &dialer{limit: []int{1, 1}} + outbox := run(t, config, d) + synctest.Wait() + assert.True(t, outbox.Send("lost")) + synctest.Wait() + failed := sent{Messages: []string{"state"}, Closed: true} + assert.Equal(t, []sent{failed}, d.streams()) + + // Retries wait between half and all of a delay that doubles from 1s. + time.Sleep(time.Second) + synctest.Wait() + assert.Equal(t, []sent{failed, {Messages: []string{"state"}}}, d.streams()) + assert.True(t, outbox.Send("lost again")) + time.Sleep(time.Second - time.Nanosecond) + synctest.Wait() + assert.Equal(t, 2, len(d.streams())) + time.Sleep(time.Second) + synctest.Wait() + assert.Equal(t, []sent{failed, failed, {Messages: []string{"state"}}}, d.streams()) + }) +} + +func TestOutboxReplacesAgedStream(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &dialer{} + run(t, stream.Config{KeepaliveInterval: time.Hour, MaxAge: 10 * time.Minute}, d) + synctest.Wait() + time.Sleep(5*time.Minute - time.Nanosecond) + synctest.Wait() + assert.Equal(t, []sent{{Messages: []string{"state"}}}, d.streams()) + + // A stream lasts between half and all of its age, and is replaced at once. + time.Sleep(5 * time.Minute) + synctest.Wait() + assert.Equal(t, []sent{{Messages: []string{"state"}, Closed: true}, {Messages: []string{"state"}}}, d.streams()) + }) +} + +func TestOutboxDropsWhenQueueIsFull(t *testing.T) { + outbox := stream.NewOutbox(config, (&dialer{}).dial, func() []string { return nil }, func() string { return "keepalive" }) + queued := 0 + for outbox.Send("event") { + queued++ + } + assert.Equal(t, 256, queued) +} diff --git a/internal/supervisor/supervisor.go b/internal/supervisor/supervisor.go index 9bab3a5..48e9c32 100644 --- a/internal/supervisor/supervisor.go +++ b/internal/supervisor/supervisor.go @@ -211,7 +211,7 @@ func (s *Supervisor) startAll(ctx context.Context) error { return nil } -// startReporter sends the mirror's heartbeats, with both proxies' quarantines, +// startReporter streams the mirror's state, with both proxies' quarantines, // to the SPECTRE server. Reporting never stops the mirror. func (s *Supervisor) startReporter(ctx context.Context) { reporter := reporter.New(s.config.Server, client.New(s.config.Server.Client), diff --git a/proto/spectre/v1/spectre.proto b/proto/spectre/v1/spectre.proto index 50da4f2..8b0b2a8 100644 --- a/proto/spectre/v1/spectre.proto +++ b/proto/spectre/v1/spectre.proto @@ -10,9 +10,9 @@ option go_package = "github.com/block/spectre/spectrepb;spectrepb"; // SpectreService collects the state of running mirrors and answers questions // about them for people and agents. service SpectreService { - // Update replaces the state of a service environment's mirror. A mirror - // sends one periodically and whenever its candidate is quarantined. - rpc Update(UpdateRequest) returns (UpdateResponse); + // Update streams a mirror's state for as long as the mirror runs. Each + // stream begins with the whole state, then changes it or keeps it alive. + rpc Update(stream UpdateRequest) returns (UpdateResponse); // ListDivergences returns the state of each mirror the server still // considers current, and the latest divergence each mirror reported. rpc ListDivergences(ListDivergencesRequest) returns (ListDivergencesResponse); @@ -31,9 +31,19 @@ enum Direction { } message UpdateRequest { - State state = 1; + oneof update { + // State replaces the mirror's whole state. Every stream begins with it. + State state = 1; + // Quarantine adds the quarantine of its direction to the state. + Quarantine quarantine = 2; + // Keepalive shows the mirror still runs when nothing has changed. + Keepalive keepalive = 3; + } } +// Keepalive carries nothing; it only keeps a quiet stream alive. +message Keepalive {} + message UpdateResponse {} // ListDivergencesRequest filters mirror states and divergences. An unset filter diff --git a/spectrepb/spectre.pb.go b/spectrepb/spectre.pb.go index bdbe307..3a52e07 100644 --- a/spectrepb/spectre.pb.go +++ b/spectrepb/spectre.pb.go @@ -140,12 +140,17 @@ func (x Quarantine_Kind) Number() protoreflect.EnumNumber { // Deprecated: Use Quarantine_Kind.Descriptor instead. func (Quarantine_Kind) EnumDescriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{7, 0} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{8, 0} } type UpdateRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - State *State `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Update: + // + // *UpdateRequest_State + // *UpdateRequest_Quarantine + // *UpdateRequest_Keepalive + Update isUpdateRequest_Update `protobuf_oneof:"update"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -180,13 +185,102 @@ func (*UpdateRequest) Descriptor() ([]byte, []int) { return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{0} } +func (x *UpdateRequest) GetUpdate() isUpdateRequest_Update { + if x != nil { + return x.Update + } + return nil +} + func (x *UpdateRequest) GetState() *State { if x != nil { - return x.State + if x, ok := x.Update.(*UpdateRequest_State); ok { + return x.State + } + } + return nil +} + +func (x *UpdateRequest) GetQuarantine() *Quarantine { + if x != nil { + if x, ok := x.Update.(*UpdateRequest_Quarantine); ok { + return x.Quarantine + } + } + return nil +} + +func (x *UpdateRequest) GetKeepalive() *Keepalive { + if x != nil { + if x, ok := x.Update.(*UpdateRequest_Keepalive); ok { + return x.Keepalive + } } return nil } +type isUpdateRequest_Update interface { + isUpdateRequest_Update() +} + +type UpdateRequest_State struct { + // State replaces the mirror's whole state. Every stream begins with it. + State *State `protobuf:"bytes,1,opt,name=state,proto3,oneof"` +} + +type UpdateRequest_Quarantine struct { + // Quarantine adds the quarantine of its direction to the state. + Quarantine *Quarantine `protobuf:"bytes,2,opt,name=quarantine,proto3,oneof"` +} + +type UpdateRequest_Keepalive struct { + // Keepalive shows the mirror still runs when nothing has changed. + Keepalive *Keepalive `protobuf:"bytes,3,opt,name=keepalive,proto3,oneof"` +} + +func (*UpdateRequest_State) isUpdateRequest_Update() {} + +func (*UpdateRequest_Quarantine) isUpdateRequest_Update() {} + +func (*UpdateRequest_Keepalive) isUpdateRequest_Update() {} + +// Keepalive carries nothing; it only keeps a quiet stream alive. +type Keepalive struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Keepalive) Reset() { + *x = Keepalive{} + mi := &file_spectre_v1_spectre_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Keepalive) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Keepalive) ProtoMessage() {} + +func (x *Keepalive) ProtoReflect() protoreflect.Message { + mi := &file_spectre_v1_spectre_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Keepalive.ProtoReflect.Descriptor instead. +func (*Keepalive) Descriptor() ([]byte, []int) { + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{1} +} + type UpdateResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -195,7 +289,7 @@ type UpdateResponse struct { func (x *UpdateResponse) Reset() { *x = UpdateResponse{} - mi := &file_spectre_v1_spectre_proto_msgTypes[1] + mi := &file_spectre_v1_spectre_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -207,7 +301,7 @@ func (x *UpdateResponse) String() string { func (*UpdateResponse) ProtoMessage() {} func (x *UpdateResponse) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[1] + mi := &file_spectre_v1_spectre_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -220,7 +314,7 @@ func (x *UpdateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateResponse.ProtoReflect.Descriptor instead. func (*UpdateResponse) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{1} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{2} } // ListDivergencesRequest filters mirror states and divergences. An unset filter @@ -238,7 +332,7 @@ type ListDivergencesRequest struct { func (x *ListDivergencesRequest) Reset() { *x = ListDivergencesRequest{} - mi := &file_spectre_v1_spectre_proto_msgTypes[2] + mi := &file_spectre_v1_spectre_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -250,7 +344,7 @@ func (x *ListDivergencesRequest) String() string { func (*ListDivergencesRequest) ProtoMessage() {} func (x *ListDivergencesRequest) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[2] + mi := &file_spectre_v1_spectre_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -263,7 +357,7 @@ func (x *ListDivergencesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListDivergencesRequest.ProtoReflect.Descriptor instead. func (*ListDivergencesRequest) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{2} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{3} } func (x *ListDivergencesRequest) GetService() string { @@ -304,7 +398,7 @@ type ListDivergencesResponse struct { func (x *ListDivergencesResponse) Reset() { *x = ListDivergencesResponse{} - mi := &file_spectre_v1_spectre_proto_msgTypes[3] + mi := &file_spectre_v1_spectre_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -316,7 +410,7 @@ func (x *ListDivergencesResponse) String() string { func (*ListDivergencesResponse) ProtoMessage() {} func (x *ListDivergencesResponse) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[3] + mi := &file_spectre_v1_spectre_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -329,7 +423,7 @@ func (x *ListDivergencesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListDivergencesResponse.ProtoReflect.Descriptor instead. func (*ListDivergencesResponse) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{3} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{4} } func (x *ListDivergencesResponse) GetStates() []*State { @@ -357,7 +451,7 @@ type ClearDivergencesRequest struct { func (x *ClearDivergencesRequest) Reset() { *x = ClearDivergencesRequest{} - mi := &file_spectre_v1_spectre_proto_msgTypes[4] + mi := &file_spectre_v1_spectre_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -369,7 +463,7 @@ func (x *ClearDivergencesRequest) String() string { func (*ClearDivergencesRequest) ProtoMessage() {} func (x *ClearDivergencesRequest) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[4] + mi := &file_spectre_v1_spectre_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -382,7 +476,7 @@ func (x *ClearDivergencesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ClearDivergencesRequest.ProtoReflect.Descriptor instead. func (*ClearDivergencesRequest) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{4} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{5} } func (x *ClearDivergencesRequest) GetService() string { @@ -407,7 +501,7 @@ type ClearDivergencesResponse struct { func (x *ClearDivergencesResponse) Reset() { *x = ClearDivergencesResponse{} - mi := &file_spectre_v1_spectre_proto_msgTypes[5] + mi := &file_spectre_v1_spectre_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -419,7 +513,7 @@ func (x *ClearDivergencesResponse) String() string { func (*ClearDivergencesResponse) ProtoMessage() {} func (x *ClearDivergencesResponse) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[5] + mi := &file_spectre_v1_spectre_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -432,7 +526,7 @@ func (x *ClearDivergencesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ClearDivergencesResponse.ProtoReflect.Descriptor instead. func (*ClearDivergencesResponse) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{5} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{6} } // Divergence is the latest quarantine of one direction of a service @@ -450,7 +544,7 @@ type Divergence struct { func (x *Divergence) Reset() { *x = Divergence{} - mi := &file_spectre_v1_spectre_proto_msgTypes[6] + mi := &file_spectre_v1_spectre_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -462,7 +556,7 @@ func (x *Divergence) String() string { func (*Divergence) ProtoMessage() {} func (x *Divergence) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[6] + mi := &file_spectre_v1_spectre_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -475,7 +569,7 @@ func (x *Divergence) ProtoReflect() protoreflect.Message { // Deprecated: Use Divergence.ProtoReflect.Descriptor instead. func (*Divergence) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{6} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{7} } func (x *Divergence) GetService() string { @@ -526,7 +620,7 @@ type Quarantine struct { func (x *Quarantine) Reset() { *x = Quarantine{} - mi := &file_spectre_v1_spectre_proto_msgTypes[7] + mi := &file_spectre_v1_spectre_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -538,7 +632,7 @@ func (x *Quarantine) String() string { func (*Quarantine) ProtoMessage() {} func (x *Quarantine) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[7] + mi := &file_spectre_v1_spectre_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -551,7 +645,7 @@ func (x *Quarantine) ProtoReflect() protoreflect.Message { // Deprecated: Use Quarantine.ProtoReflect.Descriptor instead. func (*Quarantine) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{7} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{8} } func (x *Quarantine) GetSince() *timestamppb.Timestamp { @@ -616,7 +710,7 @@ type Traffic struct { func (x *Traffic) Reset() { *x = Traffic{} - mi := &file_spectre_v1_spectre_proto_msgTypes[8] + mi := &file_spectre_v1_spectre_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -628,7 +722,7 @@ func (x *Traffic) String() string { func (*Traffic) ProtoMessage() {} func (x *Traffic) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[8] + mi := &file_spectre_v1_spectre_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -641,7 +735,7 @@ func (x *Traffic) ProtoReflect() protoreflect.Message { // Deprecated: Use Traffic.ProtoReflect.Descriptor instead. func (*Traffic) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{8} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{9} } func (x *Traffic) GetDirection() Direction { @@ -685,7 +779,7 @@ type State struct { func (x *State) Reset() { *x = State{} - mi := &file_spectre_v1_spectre_proto_msgTypes[9] + mi := &file_spectre_v1_spectre_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -697,7 +791,7 @@ func (x *State) String() string { func (*State) ProtoMessage() {} func (x *State) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[9] + mi := &file_spectre_v1_spectre_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -710,7 +804,7 @@ func (x *State) ProtoReflect() protoreflect.Message { // Deprecated: Use State.ProtoReflect.Descriptor instead. func (*State) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{9} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{10} } func (x *State) GetService() string { @@ -760,9 +854,15 @@ var File_spectre_v1_spectre_proto protoreflect.FileDescriptor const file_spectre_v1_spectre_proto_rawDesc = "" + "\n" + "\x18spectre/v1/spectre.proto\x12\n" + - "spectre.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"8\n" + - "\rUpdateRequest\x12'\n" + - "\x05state\x18\x01 \x01(\v2\x11.spectre.v1.StateR\x05state\"\x10\n" + + "spectre.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xb5\x01\n" + + "\rUpdateRequest\x12)\n" + + "\x05state\x18\x01 \x01(\v2\x11.spectre.v1.StateH\x00R\x05state\x128\n" + + "\n" + + "quarantine\x18\x02 \x01(\v2\x16.spectre.v1.QuarantineH\x00R\n" + + "quarantine\x125\n" + + "\tkeepalive\x18\x03 \x01(\v2\x15.spectre.v1.KeepaliveH\x00R\tkeepaliveB\b\n" + + "\x06update\"\v\n" + + "\tKeepalive\"\x10\n" + "\x0eUpdateResponse\"\xca\x01\n" + "\x16ListDivergencesRequest\x12\x1d\n" + "\aservice\x18\x01 \x01(\tH\x00R\aservice\x88\x01\x01\x12%\n" + @@ -821,9 +921,9 @@ const file_spectre_v1_spectre_proto_rawDesc = "" + "\tDirection\x12\x19\n" + "\x15DIRECTION_UNSPECIFIED\x10\x00\x12\x15\n" + "\x11DIRECTION_INGRESS\x10\x01\x12\x14\n" + - "\x10DIRECTION_EGRESS\x10\x022\x8c\x02\n" + - "\x0eSpectreService\x12?\n" + - "\x06Update\x12\x19.spectre.v1.UpdateRequest\x1a\x1a.spectre.v1.UpdateResponse\x12Z\n" + + "\x10DIRECTION_EGRESS\x10\x022\x8e\x02\n" + + "\x0eSpectreService\x12A\n" + + "\x06Update\x12\x19.spectre.v1.UpdateRequest\x1a\x1a.spectre.v1.UpdateResponse(\x01\x12Z\n" + "\x0fListDivergences\x12\".spectre.v1.ListDivergencesRequest\x1a#.spectre.v1.ListDivergencesResponse\x12]\n" + "\x10ClearDivergences\x12#.spectre.v1.ClearDivergencesRequest\x1a$.spectre.v1.ClearDivergencesResponseB.Z,github.com/block/spectre/spectrepb;spectrepbb\x06proto3" @@ -840,46 +940,49 @@ func file_spectre_v1_spectre_proto_rawDescGZIP() []byte { } var file_spectre_v1_spectre_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_spectre_v1_spectre_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_spectre_v1_spectre_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_spectre_v1_spectre_proto_goTypes = []any{ (Direction)(0), // 0: spectre.v1.Direction (Quarantine_Kind)(0), // 1: spectre.v1.Quarantine.Kind (*UpdateRequest)(nil), // 2: spectre.v1.UpdateRequest - (*UpdateResponse)(nil), // 3: spectre.v1.UpdateResponse - (*ListDivergencesRequest)(nil), // 4: spectre.v1.ListDivergencesRequest - (*ListDivergencesResponse)(nil), // 5: spectre.v1.ListDivergencesResponse - (*ClearDivergencesRequest)(nil), // 6: spectre.v1.ClearDivergencesRequest - (*ClearDivergencesResponse)(nil), // 7: spectre.v1.ClearDivergencesResponse - (*Divergence)(nil), // 8: spectre.v1.Divergence - (*Quarantine)(nil), // 9: spectre.v1.Quarantine - (*Traffic)(nil), // 10: spectre.v1.Traffic - (*State)(nil), // 11: spectre.v1.State - (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 13: google.protobuf.Duration + (*Keepalive)(nil), // 3: spectre.v1.Keepalive + (*UpdateResponse)(nil), // 4: spectre.v1.UpdateResponse + (*ListDivergencesRequest)(nil), // 5: spectre.v1.ListDivergencesRequest + (*ListDivergencesResponse)(nil), // 6: spectre.v1.ListDivergencesResponse + (*ClearDivergencesRequest)(nil), // 7: spectre.v1.ClearDivergencesRequest + (*ClearDivergencesResponse)(nil), // 8: spectre.v1.ClearDivergencesResponse + (*Divergence)(nil), // 9: spectre.v1.Divergence + (*Quarantine)(nil), // 10: spectre.v1.Quarantine + (*Traffic)(nil), // 11: spectre.v1.Traffic + (*State)(nil), // 12: spectre.v1.State + (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 14: google.protobuf.Duration } var file_spectre_v1_spectre_proto_depIdxs = []int32{ - 11, // 0: spectre.v1.UpdateRequest.state:type_name -> spectre.v1.State - 11, // 1: spectre.v1.ListDivergencesResponse.states:type_name -> spectre.v1.State - 8, // 2: spectre.v1.ListDivergencesResponse.divergences:type_name -> spectre.v1.Divergence - 9, // 3: spectre.v1.Divergence.quarantine:type_name -> spectre.v1.Quarantine - 12, // 4: spectre.v1.Quarantine.since:type_name -> google.protobuf.Timestamp - 1, // 5: spectre.v1.Quarantine.kind:type_name -> spectre.v1.Quarantine.Kind - 10, // 6: spectre.v1.Quarantine.traffic:type_name -> spectre.v1.Traffic - 0, // 7: spectre.v1.Quarantine.direction:type_name -> spectre.v1.Direction - 0, // 8: spectre.v1.Traffic.direction:type_name -> spectre.v1.Direction - 13, // 9: spectre.v1.State.uptime:type_name -> google.protobuf.Duration - 9, // 10: spectre.v1.State.quarantines:type_name -> spectre.v1.Quarantine - 2, // 11: spectre.v1.SpectreService.Update:input_type -> spectre.v1.UpdateRequest - 4, // 12: spectre.v1.SpectreService.ListDivergences:input_type -> spectre.v1.ListDivergencesRequest - 6, // 13: spectre.v1.SpectreService.ClearDivergences:input_type -> spectre.v1.ClearDivergencesRequest - 3, // 14: spectre.v1.SpectreService.Update:output_type -> spectre.v1.UpdateResponse - 5, // 15: spectre.v1.SpectreService.ListDivergences:output_type -> spectre.v1.ListDivergencesResponse - 7, // 16: spectre.v1.SpectreService.ClearDivergences:output_type -> spectre.v1.ClearDivergencesResponse - 14, // [14:17] is the sub-list for method output_type - 11, // [11:14] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 12, // 0: spectre.v1.UpdateRequest.state:type_name -> spectre.v1.State + 10, // 1: spectre.v1.UpdateRequest.quarantine:type_name -> spectre.v1.Quarantine + 3, // 2: spectre.v1.UpdateRequest.keepalive:type_name -> spectre.v1.Keepalive + 12, // 3: spectre.v1.ListDivergencesResponse.states:type_name -> spectre.v1.State + 9, // 4: spectre.v1.ListDivergencesResponse.divergences:type_name -> spectre.v1.Divergence + 10, // 5: spectre.v1.Divergence.quarantine:type_name -> spectre.v1.Quarantine + 13, // 6: spectre.v1.Quarantine.since:type_name -> google.protobuf.Timestamp + 1, // 7: spectre.v1.Quarantine.kind:type_name -> spectre.v1.Quarantine.Kind + 11, // 8: spectre.v1.Quarantine.traffic:type_name -> spectre.v1.Traffic + 0, // 9: spectre.v1.Quarantine.direction:type_name -> spectre.v1.Direction + 0, // 10: spectre.v1.Traffic.direction:type_name -> spectre.v1.Direction + 14, // 11: spectre.v1.State.uptime:type_name -> google.protobuf.Duration + 10, // 12: spectre.v1.State.quarantines:type_name -> spectre.v1.Quarantine + 2, // 13: spectre.v1.SpectreService.Update:input_type -> spectre.v1.UpdateRequest + 5, // 14: spectre.v1.SpectreService.ListDivergences:input_type -> spectre.v1.ListDivergencesRequest + 7, // 15: spectre.v1.SpectreService.ClearDivergences:input_type -> spectre.v1.ClearDivergencesRequest + 4, // 16: spectre.v1.SpectreService.Update:output_type -> spectre.v1.UpdateResponse + 6, // 17: spectre.v1.SpectreService.ListDivergences:output_type -> spectre.v1.ListDivergencesResponse + 8, // 18: spectre.v1.SpectreService.ClearDivergences:output_type -> spectre.v1.ClearDivergencesResponse + 16, // [16:19] is the sub-list for method output_type + 13, // [13:16] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name } func init() { file_spectre_v1_spectre_proto_init() } @@ -887,15 +990,20 @@ func file_spectre_v1_spectre_proto_init() { if File_spectre_v1_spectre_proto != nil { return } - file_spectre_v1_spectre_proto_msgTypes[2].OneofWrappers = []any{} - file_spectre_v1_spectre_proto_msgTypes[7].OneofWrappers = []any{} + file_spectre_v1_spectre_proto_msgTypes[0].OneofWrappers = []any{ + (*UpdateRequest_State)(nil), + (*UpdateRequest_Quarantine)(nil), + (*UpdateRequest_Keepalive)(nil), + } + file_spectre_v1_spectre_proto_msgTypes[3].OneofWrappers = []any{} + file_spectre_v1_spectre_proto_msgTypes[8].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_spectre_v1_spectre_proto_rawDesc), len(file_spectre_v1_spectre_proto_rawDesc)), NumEnums: 2, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, diff --git a/spectrepb/spectrepbconnect/spectre.connect.go b/spectrepb/spectrepbconnect/spectre.connect.go index 9809ac5..2840ed3 100644 --- a/spectrepb/spectrepbconnect/spectre.connect.go +++ b/spectrepb/spectrepbconnect/spectre.connect.go @@ -37,7 +37,7 @@ const ( var ( spectreServiceUpdateSpec = sync.OnceValue(func() connect.Spec { return connect.Spec{ - StreamType: connect.StreamTypeUnary, + StreamType: connect.StreamTypeClient, Schema: spectrepb.File_spectre_v1_spectre_proto.Services().ByName("SpectreService").Methods().ByName("Update"), Procedure: SpectreServiceUpdateProcedure, } @@ -60,9 +60,9 @@ var ( // SpectreServiceClient is a client for the spectre.v1.SpectreService service. type SpectreServiceClient interface { - // Update replaces the state of a service environment's mirror. A mirror - // sends one periodically and whenever its candidate is quarantined. - Update(context.Context, *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) + // Update streams a mirror's state for as long as the mirror runs. Each + // stream begins with the whole state, then changes it or keeps it alive. + Update(context.Context) (SpectreServiceUpdateClientStream, error) // ListDivergences returns the state of each mirror the server still // considers current, and the latest divergence each mirror reported. ListDivergences(context.Context, *spectrepb.ListDivergencesRequest) (*spectrepb.ListDivergencesResponse, error) @@ -77,11 +77,40 @@ func NewSpectreServiceClient(client *connect.Client) SpectreServiceClient { return &spectreServiceClient{client: client} } +// SpectreServiceUpdateClientStream is the client stream for the SpectreService's Update RPC. +type SpectreServiceUpdateClientStream struct { + stream connect.ClientStream +} + +// SendHeaders opens the stream and flushes the request headers without a message. The first Send or +// Receive does this implicitly. +func (s SpectreServiceUpdateClientStream) SendHeaders() error { + return s.stream.SendHeaders() +} + +// Send sends a request message to the server. +func (s SpectreServiceUpdateClientStream) Send(req *spectrepb.UpdateRequest) error { + return s.stream.Send(req) +} + +// CloseAndReceive closes the request side of the stream and returns the single response message. It +// reads the stream to completion to release its resources. +func (s SpectreServiceUpdateClientStream) CloseAndReceive() (*spectrepb.UpdateResponse, error) { + if err := s.stream.CloseSend(); err != nil { + return nil, err + } + var res spectrepb.UpdateResponse + if err := s.stream.Receive(&res); err != nil { + return nil, err + } + return &res, nil +} + // SpectreServiceHandler is an implementation of the spectre.v1.SpectreService service. type SpectreServiceHandler interface { - // Update replaces the state of a service environment's mirror. A mirror - // sends one periodically and whenever its candidate is quarantined. - Update(context.Context, *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) + // Update streams a mirror's state for as long as the mirror runs. Each + // stream begins with the whole state, then changes it or keeps it alive. + Update(context.Context, SpectreServiceUpdateServerStream) (*spectrepb.UpdateResponse, error) // ListDivergences returns the state of each mirror the server still // considers current, and the latest divergence each mirror reported. ListDivergences(context.Context, *spectrepb.ListDivergencesRequest) (*spectrepb.ListDivergencesResponse, error) @@ -101,10 +130,24 @@ func RegisterSpectreServiceHandler(server *connect.Server, svc SpectreServiceHan ) } +// SpectreServiceUpdateServerStream is the server stream for the SpectreService's Update RPC. +type SpectreServiceUpdateServerStream struct { + stream connect.ServerStream +} + +// Receive returns the next request message from the client. +func (s SpectreServiceUpdateServerStream) Receive() (*spectrepb.UpdateRequest, error) { + var req spectrepb.UpdateRequest + if err := s.stream.Receive(&req); err != nil { + return nil, err + } + return &req, nil +} + // UnimplementedSpectreServiceHandler returns CodeUnimplemented from all methods. type UnimplementedSpectreServiceHandler struct{} -func (UnimplementedSpectreServiceHandler) Update(context.Context, *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) { +func (UnimplementedSpectreServiceHandler) Update(context.Context, SpectreServiceUpdateServerStream) (*spectrepb.UpdateResponse, error) { return nil, connect.NewError(connect.CodeUnimplemented, "spectre.v1.SpectreService.Update is not implemented") } @@ -120,12 +163,12 @@ type spectreServiceClient struct { client *connect.Client } -func (c *spectreServiceClient) Update(ctx context.Context, req *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) { - var res spectrepb.UpdateResponse - if err := c.client.CallUnary(ctx, spectreServiceUpdateSpec(), req, &res); err != nil { - return nil, err +func (c *spectreServiceClient) Update(ctx context.Context) (SpectreServiceUpdateClientStream, error) { + stream, err := c.client.CallClientStream(ctx, spectreServiceUpdateSpec()) + if err != nil { + return SpectreServiceUpdateClientStream{}, err } - return &res, nil + return SpectreServiceUpdateClientStream{stream: stream}, nil } func (c *spectreServiceClient) ListDivergences(ctx context.Context, req *spectrepb.ListDivergencesRequest) (*spectrepb.ListDivergencesResponse, error) { @@ -147,11 +190,7 @@ func (c *spectreServiceClient) ClearDivergences(ctx context.Context, req *spectr type spectreServiceHandler struct{ svc SpectreServiceHandler } func (h spectreServiceHandler) update(ctx context.Context, _ connect.Spec, stream connect.ServerStream) error { - var req spectrepb.UpdateRequest - if err := stream.Receive(&req); err != nil { - return err - } - res, err := h.svc.Update(ctx, &req) + res, err := h.svc.Update(ctx, SpectreServiceUpdateServerStream{stream: stream}) if err != nil { return err } From 1bac2a3808cab9a2a19c09608947c20861ebf6bc Mon Sep 17 00:00:00 2001 From: Juho Makinen Date: Wed, 4 Nov 2026 14:38:56 +1100 Subject: [PATCH 2/5] refactor(stream): send keepalives from the stream, not as an update Keeping a quiet stream alive is the stream's concern, so every kind of message a stream carries would otherwise need its own keepalive variant. Co-Authored-By: Claude Opus 5.5 --- internal/model/update.go | 5 ----- internal/server/client/client.go | 6 ++++++ internal/server/reporter/reporter.go | 3 +-- internal/server/service.go | 22 +++++++++++++--------- internal/server/wire/wire.go | 6 ++---- internal/stream/outbox.go | 28 +++++++++++++++------------- internal/stream/outbox_test.go | 8 ++++++-- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/internal/model/update.go b/internal/model/update.go index 4d705f0..399c44b 100644 --- a/internal/model/update.go +++ b/internal/model/update.go @@ -13,8 +13,3 @@ type QuarantineUpdate struct { } func (QuarantineUpdate) update() {} - -// Keepalive shows the server that a quiet mirror still runs. -type Keepalive struct{} - -func (Keepalive) update() {} diff --git a/internal/server/client/client.go b/internal/server/client/client.go index 55ed08f..7dfc815 100644 --- a/internal/server/client/client.go +++ b/internal/server/client/client.go @@ -95,6 +95,12 @@ func (u *Updates) Send(update model.Update) error { return errors.Wrap(u.stream.Send(wire.UpdateMessage(update)), "send update") } +// Keepalive shows the server that a quiet mirror is still connected. +func (u *Updates) Keepalive() error { + keepalive := &pb.UpdateRequest{Update: &pb.UpdateRequest_Keepalive{Keepalive: &pb.Keepalive{}}} + return errors.Wrap(u.stream.Send(keepalive), "send keepalive") +} + // CloseAndReceive ends the stream and returns the error the server ended it with. func (u *Updates) CloseAndReceive() error { _, err := u.stream.CloseAndReceive() diff --git a/internal/server/reporter/reporter.go b/internal/server/reporter/reporter.go index e2d364c..8f62416 100644 --- a/internal/server/reporter/reporter.go +++ b/internal/server/reporter/reporter.go @@ -54,8 +54,7 @@ func (r *Reporter) Run(ctx context.Context) { return r.client.OpenUpdates(ctx) } resync := func() []model.Update { return []model.Update{r.state()} } - keepalive := func() model.Update { return model.Keepalive{} } - outbox := stream.NewOutbox(r.config.Stream, dial, resync, keepalive) + outbox := stream.NewOutbox(r.config.Stream, dial, resync) for _, source := range r.sources { go r.report(ctx, source, outbox) } diff --git a/internal/server/service.go b/internal/server/service.go index dd4c7ce..f508b9a 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -163,23 +163,27 @@ func Serve(ctx context.Context, config ServeConfig) error { } // nextState applies a streamed update to the state a stream has built. A -// stream must begin with the whole state. +// stream must begin with the whole state, and a keepalive leaves it as it is. func nextState(state Option[model.State], message *pb.UpdateRequest) (model.State, error) { + current, started := state.Get() + if message.GetKeepalive() != nil { + if !started { + return model.State{}, errors.New("an update stream must begin with the mirror's state") + } + return current, nil + } update, err := wire.UpdateFromMessage(message) if err != nil { return model.State{}, errors.WithStack(err) } - if whole, ok := update.(model.State); ok { - return whole, nil - } - current, ok := state.Get() - if !ok { - return model.State{}, errors.New("an update stream must begin with the mirror's state") - } switch update := update.(type) { + case model.State: + return update, nil case model.QuarantineUpdate: + if !started { + return model.State{}, errors.New("an update stream must begin with the mirror's state") + } return current.Quarantined(update.Direction, update.Quarantine), nil - case model.State, model.Keepalive: } return current, nil } diff --git a/internal/server/wire/wire.go b/internal/server/wire/wire.go index 0db875b..ca710b0 100644 --- a/internal/server/wire/wire.go +++ b/internal/server/wire/wire.go @@ -71,13 +71,11 @@ func UpdateMessage(update model.Update) *pb.UpdateRequest { return &pb.UpdateRequest{Update: &pb.UpdateRequest_State{State: StateMessage(update)}} case model.QuarantineUpdate: return &pb.UpdateRequest{Update: &pb.UpdateRequest_Quarantine{Quarantine: QuarantineMessage(update.Direction, update.Quarantine)}} - case model.Keepalive: - return &pb.UpdateRequest{Update: &pb.UpdateRequest_Keepalive{Keepalive: &pb.Keepalive{}}} } return &pb.UpdateRequest{} } -// UpdateFromMessage restores a streamed update. +// UpdateFromMessage restores a streamed update. A keepalive holds none. func UpdateFromMessage(message *pb.UpdateRequest) (model.Update, error) { switch update := message.GetUpdate().(type) { case *pb.UpdateRequest_State: @@ -89,7 +87,7 @@ func UpdateFromMessage(message *pb.UpdateRequest) (model.Update, error) { } return model.QuarantineUpdate{Direction: direction, Quarantine: quarantine}, nil case *pb.UpdateRequest_Keepalive: - return model.Keepalive{}, nil + return nil, errors.New("a keepalive holds no update") } return nil, errors.New("update holds nothing") } diff --git a/internal/stream/outbox.go b/internal/stream/outbox.go index a0a5f5d..cd1f2fc 100644 --- a/internal/stream/outbox.go +++ b/internal/stream/outbox.go @@ -2,7 +2,8 @@ // stream from a client open through failures and sends it what producers queue. // Every stream it opens begins with a resync of the client's whole state, so // only events need queueing, and a stream may fail at any time without losing -// state. EndEarly lets a server end such a stream before the client does. +// state. A quiet stream sends keepalives, which carry no message. EndEarly +// lets a server end such a stream before the client does. package stream import ( @@ -19,6 +20,8 @@ import ( type Conn[M any] interface { // Send sends message. It fails once the server has ended the stream. Send(message M) error + // Keepalive shows the server that a quiet client is still connected. + Keepalive() error // CloseAndReceive ends the stream and returns the error the server ended it with. CloseAndReceive() error } @@ -48,17 +51,16 @@ const queueSize = 256 // Outbox keeps one stream open and sends it the messages producers queue. type Outbox[M any] struct { - config Config - dial func(ctx context.Context) (Conn[M], error) - resync func() []M - keepalive func() M - queue chan M + config Config + dial func(ctx context.Context) (Conn[M], error) + resync func() []M + queue chan M } // NewOutbox returns an outbox that opens streams with dial. Each stream begins -// with the messages resync returns, and a quiet stream sends keepalive. -func NewOutbox[M any](config Config, dial func(ctx context.Context) (Conn[M], error), resync func() []M, keepalive func() M) *Outbox[M] { - return &Outbox[M]{config: config, dial: dial, resync: resync, keepalive: keepalive, queue: make(chan M, queueSize)} +// with the messages resync returns. +func NewOutbox[M any](config Config, dial func(ctx context.Context) (Conn[M], error), resync func() []M) *Outbox[M] { + return &Outbox[M]{config: config, dial: dial, resync: resync, queue: make(chan M, queueSize)} } // Send queues message for the stream without blocking. It drops the message @@ -119,7 +121,6 @@ func (o *Outbox[M]) stream(ctx context.Context) error { age := time.NewTimer(jitter(o.config.MaxAge)) defer age.Stop() for { - var message M select { case <-ctx.Done(): cancel() @@ -128,10 +129,11 @@ func (o *Outbox[M]) stream(ctx context.Context) error { case <-age.C: return errors.Wrap(conn.CloseAndReceive(), "close aged stream") case <-keepalive.C: - message = o.keepalive() - case message = <-o.queue: + err = conn.Keepalive() + case message := <-o.queue: + err = conn.Send(message) } - if err := conn.Send(message); err != nil { + if err != nil { return failed(conn, err) } keepalive.Reset(o.config.KeepaliveInterval) diff --git a/internal/stream/outbox_test.go b/internal/stream/outbox_test.go index b7d54a7..e35b3f0 100644 --- a/internal/stream/outbox_test.go +++ b/internal/stream/outbox_test.go @@ -33,6 +33,10 @@ func (c *conn) Send(message string) error { return nil } +func (c *conn) Keepalive() error { + return c.Send("keepalive") +} + func (c *conn) CloseAndReceive() error { c.mu.Lock() defer c.mu.Unlock() @@ -82,7 +86,7 @@ func (d *dialer) streams() []sent { // run runs an outbox until the test ends. func run(t *testing.T, config stream.Config, d *dialer) *stream.Outbox[string] { t.Helper() - outbox := stream.NewOutbox(config, d.dial, func() []string { return []string{"state"} }, func() string { return "keepalive" }) + outbox := stream.NewOutbox(config, d.dial, func() []string { return []string{"state"} }) ctx, cancel := context.WithCancel(t.Context()) go outbox.Run(ctx) t.Cleanup(func() { @@ -156,7 +160,7 @@ func TestOutboxReplacesAgedStream(t *testing.T) { } func TestOutboxDropsWhenQueueIsFull(t *testing.T) { - outbox := stream.NewOutbox(config, (&dialer{}).dial, func() []string { return nil }, func() string { return "keepalive" }) + outbox := stream.NewOutbox(config, (&dialer{}).dial, func() []string { return nil }) queued := 0 for outbox.Send("event") { queued++ From 9ddb320dd1a61599ef58de2f5109579047c851a4 Mon Sep 17 00:00:00 2001 From: Juho Makinen Date: Wed, 4 Nov 2026 15:03:51 +1100 Subject: [PATCH 3/5] refactor(stream): name the outbox OutChannel OutChannel says what it is: the outgoing half of a channel to the server, which keeps its stream open, rather than a mailbox messages wait in. Co-Authored-By: Claude Opus 5.5 --- internal/server/reporter/reporter.go | 10 ++--- internal/stream/{outbox.go => outchannel.go} | 36 +++++++++--------- .../{outbox_test.go => outchannel_test.go} | 38 +++++++++---------- 3 files changed, 42 insertions(+), 42 deletions(-) rename internal/stream/{outbox.go => outchannel.go} (79%) rename internal/stream/{outbox_test.go => outchannel_test.go} (80%) diff --git a/internal/server/reporter/reporter.go b/internal/server/reporter/reporter.go index 8f62416..cff52f8 100644 --- a/internal/server/reporter/reporter.go +++ b/internal/server/reporter/reporter.go @@ -54,22 +54,22 @@ func (r *Reporter) Run(ctx context.Context) { return r.client.OpenUpdates(ctx) } resync := func() []model.Update { return []model.Update{r.state()} } - outbox := stream.NewOutbox(r.config.Stream, dial, resync) + channel := stream.NewOutChannel(r.config.Stream, dial, resync) for _, source := range r.sources { - go r.report(ctx, source, outbox) + go r.report(ctx, source, channel) } - outbox.Run(ctx) + channel.Run(ctx) } // report sends the source's quarantine once it begins. The state holds it too, // so a stream that fails before sending it resyncs it. -func (r *Reporter) report(ctx context.Context, source Source, outbox *stream.Outbox[model.Update]) { +func (r *Reporter) report(ctx context.Context, source Source, channel *stream.OutChannel[model.Update]) { select { case quarantine := <-source.Quarantined: r.mu.Lock() r.quarantines[source.Direction] = quarantine r.mu.Unlock() - if !outbox.Send(model.QuarantineUpdate{Direction: source.Direction, Quarantine: quarantine}) { + if !channel.Send(model.QuarantineUpdate{Direction: source.Direction, Quarantine: quarantine}) { logger.FromContext(ctx).WarnContext(ctx, "Dropped a quarantine update for the SPECTRE server; the next stream resyncs it") } case <-ctx.Done(): diff --git a/internal/stream/outbox.go b/internal/stream/outchannel.go similarity index 79% rename from internal/stream/outbox.go rename to internal/stream/outchannel.go index cd1f2fc..e167347 100644 --- a/internal/stream/outbox.go +++ b/internal/stream/outchannel.go @@ -1,4 +1,4 @@ -// Package stream keeps long-lived client streams open. An Outbox keeps one +// Package stream keeps long-lived client streams open. An OutChannel keeps one // stream from a client open through failures and sends it what producers queue. // Every stream it opens begins with a resync of the client's whole state, so // only events need queueing, and a stream may fail at any time without losing @@ -26,7 +26,7 @@ type Conn[M any] interface { CloseAndReceive() error } -// Config configures how an outbox keeps its stream open. +// Config configures how a channel keeps its stream open. type Config struct { KeepaliveInterval time.Duration `default:"30s" help:"How often to send a keepalive on a stream that is otherwise quiet."` MaxAge time.Duration `name:"stream-max-age" default:"10m" help:"How long a stream lasts, with jitter, before it is replaced, so streams spread across servers."` @@ -49,25 +49,25 @@ const ( // queueSize is how many messages wait for a stream before Send drops them. const queueSize = 256 -// Outbox keeps one stream open and sends it the messages producers queue. -type Outbox[M any] struct { +// OutChannel keeps one stream open and sends it the messages producers queue. +type OutChannel[M any] struct { config Config dial func(ctx context.Context) (Conn[M], error) resync func() []M queue chan M } -// NewOutbox returns an outbox that opens streams with dial. Each stream begins +// NewOutChannel returns a channel that opens streams with dial. Each stream begins // with the messages resync returns. -func NewOutbox[M any](config Config, dial func(ctx context.Context) (Conn[M], error), resync func() []M) *Outbox[M] { - return &Outbox[M]{config: config, dial: dial, resync: resync, queue: make(chan M, queueSize)} +func NewOutChannel[M any](config Config, dial func(ctx context.Context) (Conn[M], error), resync func() []M) *OutChannel[M] { + return &OutChannel[M]{config: config, dial: dial, resync: resync, queue: make(chan M, queueSize)} } // Send queues message for the stream without blocking. It drops the message // when the queue is full. A queued message is lost if its stream fails. -func (o *Outbox[M]) Send(message M) (queued bool) { +func (c *OutChannel[M]) Send(message M) (queued bool) { select { - case o.queue <- message: + case c.queue <- message: return true default: return false @@ -76,11 +76,11 @@ func (o *Outbox[M]) Send(message M) (queued bool) { // Run keeps a stream open until ctx ends, replacing it when it fails, with // backoff, or when it reaches its age. Failures are only logged. -func (o *Outbox[M]) Run(ctx context.Context) { +func (c *OutChannel[M]) Run(ctx context.Context) { delay := minRetryDelay for { opened := time.Now() - err := o.stream(ctx) + err := c.stream(ctx) if ctx.Err() != nil { return } @@ -104,21 +104,21 @@ func (o *Outbox[M]) Run(ctx context.Context) { // stream sends on one stream until it fails or ctx ends. It returns nil once // the stream reaches its age and the server accepts it closing. -func (o *Outbox[M]) stream(ctx context.Context) error { +func (c *OutChannel[M]) stream(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - conn, err := o.dial(ctx) + conn, err := c.dial(ctx) if err != nil { return errors.Wrap(err, "open stream") } - for _, message := range o.resync() { + for _, message := range c.resync() { if err := conn.Send(message); err != nil { return failed(conn, err) } } - keepalive := time.NewTimer(o.config.KeepaliveInterval) + keepalive := time.NewTimer(c.config.KeepaliveInterval) defer keepalive.Stop() - age := time.NewTimer(jitter(o.config.MaxAge)) + age := time.NewTimer(jitter(c.config.MaxAge)) defer age.Stop() for { select { @@ -130,13 +130,13 @@ func (o *Outbox[M]) stream(ctx context.Context) error { return errors.Wrap(conn.CloseAndReceive(), "close aged stream") case <-keepalive.C: err = conn.Keepalive() - case message := <-o.queue: + case message := <-c.queue: err = conn.Send(message) } if err != nil { return failed(conn, err) } - keepalive.Reset(o.config.KeepaliveInterval) + keepalive.Reset(c.config.KeepaliveInterval) } } diff --git a/internal/stream/outbox_test.go b/internal/stream/outchannel_test.go similarity index 80% rename from internal/stream/outbox_test.go rename to internal/stream/outchannel_test.go index e35b3f0..4f25a02 100644 --- a/internal/stream/outbox_test.go +++ b/internal/stream/outchannel_test.go @@ -14,7 +14,7 @@ import ( "github.com/block/spectre/internal/stream" ) -// conn records what an outbox sends on one stream. +// conn records what a channel sends on one stream. type conn struct { mu sync.Mutex sent []string @@ -53,7 +53,7 @@ type sent struct { Closed bool } -// dialer hands out a new conn for every stream an outbox opens. +// dialer hands out a new conn for every stream a channel opens. type dialer struct { mu sync.Mutex conns []*conn @@ -83,32 +83,32 @@ func (d *dialer) streams() []sent { return streams } -// run runs an outbox until the test ends. -func run(t *testing.T, config stream.Config, d *dialer) *stream.Outbox[string] { +// run runs a channel until the test ends. +func run(t *testing.T, config stream.Config, d *dialer) *stream.OutChannel[string] { t.Helper() - outbox := stream.NewOutbox(config, d.dial, func() []string { return []string{"state"} }) + channel := stream.NewOutChannel(config, d.dial, func() []string { return []string{"state"} }) ctx, cancel := context.WithCancel(t.Context()) - go outbox.Run(ctx) + go channel.Run(ctx) t.Cleanup(func() { cancel() synctest.Wait() }) - return outbox + return channel } var config = stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: time.Hour} -func TestOutboxResyncsThenSendsAndKeepsAlive(t *testing.T) { +func TestOutChannelResyncsThenSendsAndKeepsAlive(t *testing.T) { synctest.Test(t, func(t *testing.T) { d := &dialer{} - outbox := run(t, config, d) - assert.True(t, outbox.Send("event")) + channel := run(t, config, d) + assert.True(t, channel.Send("event")) synctest.Wait() assert.Equal(t, []sent{{Messages: []string{"state", "event"}}}, d.streams()) // Only a quiet stream sends keepalives. time.Sleep(20 * time.Second) - assert.True(t, outbox.Send("later")) + assert.True(t, channel.Send("later")) time.Sleep(20 * time.Second) synctest.Wait() assert.Equal(t, []sent{{Messages: []string{"state", "event", "later"}}}, d.streams()) @@ -118,13 +118,13 @@ func TestOutboxResyncsThenSendsAndKeepsAlive(t *testing.T) { }) } -func TestOutboxReconnectsWithBackoff(t *testing.T) { +func TestOutChannelReconnectsWithBackoff(t *testing.T) { synctest.Test(t, func(t *testing.T) { // The first two streams end after the resync, the third lasts. d := &dialer{limit: []int{1, 1}} - outbox := run(t, config, d) + channel := run(t, config, d) synctest.Wait() - assert.True(t, outbox.Send("lost")) + assert.True(t, channel.Send("lost")) synctest.Wait() failed := sent{Messages: []string{"state"}, Closed: true} assert.Equal(t, []sent{failed}, d.streams()) @@ -133,7 +133,7 @@ func TestOutboxReconnectsWithBackoff(t *testing.T) { time.Sleep(time.Second) synctest.Wait() assert.Equal(t, []sent{failed, {Messages: []string{"state"}}}, d.streams()) - assert.True(t, outbox.Send("lost again")) + assert.True(t, channel.Send("lost again")) time.Sleep(time.Second - time.Nanosecond) synctest.Wait() assert.Equal(t, 2, len(d.streams())) @@ -143,7 +143,7 @@ func TestOutboxReconnectsWithBackoff(t *testing.T) { }) } -func TestOutboxReplacesAgedStream(t *testing.T) { +func TestOutChannelReplacesAgedStream(t *testing.T) { synctest.Test(t, func(t *testing.T) { d := &dialer{} run(t, stream.Config{KeepaliveInterval: time.Hour, MaxAge: 10 * time.Minute}, d) @@ -159,10 +159,10 @@ func TestOutboxReplacesAgedStream(t *testing.T) { }) } -func TestOutboxDropsWhenQueueIsFull(t *testing.T) { - outbox := stream.NewOutbox(config, (&dialer{}).dial, func() []string { return nil }) +func TestOutChannelDropsWhenQueueIsFull(t *testing.T) { + channel := stream.NewOutChannel(config, (&dialer{}).dial, func() []string { return nil }) queued := 0 - for outbox.Send("event") { + for channel.Send("event") { queued++ } assert.Equal(t, 256, queued) From 9ef06c0fa9a4cffe6261018adaad367fd01fb28a Mon Sep 17 00:00:00 2001 From: Juho Makinen Date: Wed, 4 Nov 2026 15:04:27 +1100 Subject: [PATCH 4/5] docs: say why the keepalive interval must stay under 60s In staging, Caddy ended request streams that were quiet for 60s, so a longer keepalive interval would drop every quiet mirror's stream. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb14a7a..7b1881b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,10 +87,12 @@ host, such as `--server-url=http://spectre.mesh --server-socket=@egress.sock`. A mirror sends everything to the server over one stream, which begins with its whole state. It then sends each quarantine as it begins, and a keepalive every -`--server-keepalive-interval` (30s by default) when nothing else is sent. When -the stream fails, the mirror logs it and reconnects with backoff, from 1s up to -1m. It also replaces the stream after `--server-stream-max-age` (10m by -default, with jitter), so streams spread across server replicas. +`--server-keepalive-interval` (30s by default) when nothing else is sent. +Keep the interval below the idle timeout of any proxy in front of the server; +Caddy, for one, ends a request stream after 60s without data. When the stream +fails, the mirror logs it and reconnects with backoff, from 1s up to 1m. It also +replaces the stream after `--server-stream-max-age` (10m by default, with +jitter), so streams spread across server replicas. List the reporting mirrors and recorded divergences with `spectre status --server-url=URL`. Filter with `--service`, From 695ca52e1224923ecae90c4a10ed441e2eeaa2bb Mon Sep 17 00:00:00 2001 From: Juho Makinen Date: Wed, 4 Nov 2026 15:29:02 +1100 Subject: [PATCH 5/5] refactor(reporter): keep the stream in the reporter, without generics The reporter is the only stream a mirror opens, so a generic channel abstracted over one caller. Captures can pull out an abstraction once they are a second caller that shows what it needs. Each source sends its quarantine once, so the queue never fills and the reporter's run loop owns the quarantines without a lock. Co-Authored-By: Claude Opus 5.5 --- cli_internal_test.go | 23 +-- .../middleware/endearly}/handler.go | 7 +- internal/server/reporter/reporter.go | 153 ++++++++++++---- internal/server/reporter/reporter_test.go | 110 ++++++++---- internal/server/service.go | 4 +- internal/stream/outchannel.go | 155 ---------------- internal/stream/outchannel_test.go | 169 ------------------ internal/supervisor/config.go | 3 + 8 files changed, 216 insertions(+), 408 deletions(-) rename internal/{stream => httpserver/middleware/endearly}/handler.go (85%) delete mode 100644 internal/stream/outchannel.go delete mode 100644 internal/stream/outchannel_test.go diff --git a/cli_internal_test.go b/cli_internal_test.go index b11c4b5..a540193 100644 --- a/cli_internal_test.go +++ b/cli_internal_test.go @@ -17,7 +17,6 @@ import ( "github.com/block/spectre/internal/server/client" "github.com/block/spectre/internal/server/reporter" "github.com/block/spectre/internal/server/storage" - "github.com/block/spectre/internal/stream" "github.com/block/spectre/internal/supervisor" ) @@ -29,11 +28,12 @@ func reportingFlags() []string { // expectedReporting is the reporter config that reportingFlags parse to. func expectedReporting() reporter.Config { return reporter.Config{ - Client: client.Config{URL: "http://server.example"}, - Service: "users", - Environment: "production", - Revision: "abc123", - Stream: stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: 10 * time.Minute}, + Client: client.Config{URL: "http://server.example"}, + Service: "users", + Environment: "production", + Revision: "abc123", + KeepaliveInterval: 30 * time.Second, + StreamMaxAge: 10 * time.Minute, } } @@ -86,11 +86,12 @@ func TestSuperviseCommandReadsFlagsFromEnvironment(t *testing.T) { }) assert.NoError(t, err) expected := reporter.Config{ - Client: client.Config{URL: "http://server.example"}, - Service: "invoices", - Environment: "production-us-west-2", - Revision: "3f9c2a1", - Stream: stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: 10 * time.Minute}, + Client: client.Config{URL: "http://server.example"}, + Service: "invoices", + Environment: "production-us-west-2", + Revision: "3f9c2a1", + KeepaliveInterval: 30 * time.Second, + StreamMaxAge: 10 * time.Minute, } assert.Equal(t, expected, command.Supervise.Supervisor.Server) } diff --git a/internal/stream/handler.go b/internal/httpserver/middleware/endearly/handler.go similarity index 85% rename from internal/stream/handler.go rename to internal/httpserver/middleware/endearly/handler.go index 85c2a36..abe72ab 100644 --- a/internal/stream/handler.go +++ b/internal/httpserver/middleware/endearly/handler.go @@ -1,4 +1,5 @@ -package stream +// Package endearly lets a server end a client stream before the client does. +package endearly import ( "io" @@ -8,9 +9,9 @@ import ( "github.com/alecthomas/errors" ) -// EndEarly lets next end a client stream before the client does. Otherwise an +// New lets next end a client stream before the client does. Otherwise an // HTTP/1 server reads the rest of the request before it replies. -func EndEarly(next http.Handler) http.Handler { +func New(next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { request.Body = newUnendedBody(request.Body, http.NewResponseController(writer)) next.ServeHTTP(writer, request) diff --git a/internal/server/reporter/reporter.go b/internal/server/reporter/reporter.go index cff52f8..44c2a4f 100644 --- a/internal/server/reporter/reporter.go +++ b/internal/server/reporter/reporter.go @@ -1,29 +1,48 @@ -// Package reporter streams what a mirror sends the SPECTRE server: its state -// and every change to it, over one stream. +// Package reporter streams what a mirror sends the SPECTRE server over one +// stream. Every stream begins with the mirror's whole state, then sends each +// quarantine as it begins, so a stream may fail at any time without losing +// state. A quiet stream sends keepalives. A failed stream is replaced with +// backoff, and a stream is replaced when it ages, so streams spread across +// servers. package reporter import ( "context" - "maps" - "sync" + "math/rand/v2" "time" + "github.com/alecthomas/errors" + "github.com/block/spectre/internal" "github.com/block/spectre/internal/logger" "github.com/block/spectre/internal/model" "github.com/block/spectre/internal/server/client" - "github.com/block/spectre/internal/stream" ) // Config configures how a mirror reports to the SPECTRE server. type Config struct { - Client client.Config `embed:""` - Stream stream.Config `embed:"" prefix:"server-"` - Service string `required:"" help:"Service the mirror runs, reported to the SPECTRE server."` - Environment string `required:"" help:"Environment the mirror runs in, such as production-us-west-2, reported to the SPECTRE server."` - Revision string `required:"" help:"Commit of the candidate the mirror runs, reported to the SPECTRE server."` + Client client.Config `embed:""` + KeepaliveInterval time.Duration `name:"server-keepalive-interval" default:"30s" help:"How often to send a keepalive on a stream to the SPECTRE server that is otherwise quiet."` + StreamMaxAge time.Duration `name:"server-stream-max-age" default:"10m" help:"How long a stream to the SPECTRE server lasts, with jitter, before it is replaced, so streams spread across servers."` + Service string `required:"" help:"Service the mirror runs, reported to the SPECTRE server."` + Environment string `required:"" help:"Environment the mirror runs in, such as production-us-west-2, reported to the SPECTRE server."` + Revision string `required:"" help:"Commit of the candidate the mirror runs, reported to the SPECTRE server."` +} + +// Validate checks that the stream intervals are positive. +func (c Config) Validate() error { + if c.KeepaliveInterval <= 0 || c.StreamMaxAge <= 0 { + return errors.New("server keepalive interval and stream max age must be positive") + } + return nil } +// Retries of a failed stream back off between these delays. +const ( + minRetryDelay = time.Second + maxRetryDelay = time.Minute +) + // Source is a proxy whose quarantine a mirror reports. type Source struct { Direction model.Direction @@ -37,8 +56,7 @@ type Reporter struct { client *client.Client sources []Source started time.Time - - mu sync.Mutex + // quarantines belongs to the goroutine that runs streams. quarantines map[model.Direction]model.Quarantine } @@ -47,45 +65,108 @@ func New(config Config, server *client.Client, sources ...Source) *Reporter { return &Reporter{config: config, client: server, sources: sources, started: time.Now(), quarantines: map[model.Direction]model.Quarantine{}} } -// Run streams the mirror's state until ctx is cancelled. Every stream begins -// with the whole state, then sends each quarantine as it begins. +// Run keeps a stream open until ctx ends, replacing it when it fails, with +// backoff, or when it ages. Failures are only logged. func (r *Reporter) Run(ctx context.Context) { - dial := func(ctx context.Context) (stream.Conn[model.Update], error) { - return r.client.OpenUpdates(ctx) - } - resync := func() []model.Update { return []model.Update{r.state()} } - channel := stream.NewOutChannel(r.config.Stream, dial, resync) + // Each source sends once, so sending never blocks. + quarantines := make(chan model.QuarantineUpdate, len(r.sources)) for _, source := range r.sources { - go r.report(ctx, source, channel) + go forward(ctx, source, quarantines) + } + delay := minRetryDelay + for { + opened := time.Now() + err := r.stream(ctx, quarantines) + if ctx.Err() != nil { + return + } + // Only consecutive quick failures back off further. + if err == nil || time.Since(opened) > maxRetryDelay { + delay = minRetryDelay + } + if err == nil { + continue + } + wait := jitter(delay) + logger.FromContext(ctx).WarnContext(ctx, "Lost the stream to the SPECTRE server", "error", err, "retry", wait) + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + delay = min(delay*2, maxRetryDelay) } - channel.Run(ctx) } -// report sends the source's quarantine once it begins. The state holds it too, -// so a stream that fails before sending it resyncs it. -func (r *Reporter) report(ctx context.Context, source Source, channel *stream.OutChannel[model.Update]) { - select { - case quarantine := <-source.Quarantined: - r.mu.Lock() - r.quarantines[source.Direction] = quarantine - r.mu.Unlock() - if !channel.Send(model.QuarantineUpdate{Direction: source.Direction, Quarantine: quarantine}) { - logger.FromContext(ctx).WarnContext(ctx, "Dropped a quarantine update for the SPECTRE server; the next stream resyncs it") +// stream sends on one stream until it fails or ctx ends. It returns nil once +// the stream ages and the server accepts it closing. +func (r *Reporter) stream(ctx context.Context, quarantines <-chan model.QuarantineUpdate) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + updates, err := r.client.OpenUpdates(ctx) + if err != nil { + return errors.Wrap(err, "open stream") + } + if err := updates.Send(r.state()); err != nil { + return failed(updates, err) + } + keepalive := time.NewTimer(r.config.KeepaliveInterval) + defer keepalive.Stop() + age := time.NewTimer(jitter(r.config.StreamMaxAge)) + defer age.Stop() + for { + select { + case <-ctx.Done(): + cancel() + _ = updates.CloseAndReceive() //nolint:errcheck // Cancelling has already aborted the stream. + return errors.WithStack(ctx.Err()) + case <-age.C: + return errors.Wrap(updates.CloseAndReceive(), "close aged stream") + case <-keepalive.C: + err = updates.Keepalive() + case update := <-quarantines: + // Recorded first, so the next stream's state holds it if this send fails. + r.quarantines[update.Direction] = update.Quarantine + err = updates.Send(update) } - case <-ctx.Done(): + if err != nil { + return failed(updates, err) + } + keepalive.Reset(r.config.KeepaliveInterval) } } -// state returns the mirror's whole state. +// state returns the mirror's whole state, to send at once. func (r *Reporter) state() model.State { - r.mu.Lock() - defer r.mu.Unlock() return model.State{ Service: r.config.Service, Environment: r.config.Environment, Revision: r.config.Revision, SpectreVersion: internal.Version, Uptime: time.Since(r.started), - Quarantines: maps.Clone(r.quarantines), + Quarantines: r.quarantines, } } + +// forward sends the source's quarantine to quarantines once it begins. +func forward(ctx context.Context, source Source, quarantines chan<- model.QuarantineUpdate) { + select { + case quarantine := <-source.Quarantined: + quarantines <- model.QuarantineUpdate{Direction: source.Direction, Quarantine: quarantine} + case <-ctx.Done(): + } +} + +// failed closes a stream whose send failed. A send fails once the server has +// ended the stream, and closing returns why it did. +func failed(updates *client.Updates, err error) error { + if closeErr := updates.CloseAndReceive(); closeErr != nil { + return errors.Wrap(closeErr, "stream ended") + } + return errors.Wrap(err, "send on stream") +} + +// jitter returns a random duration between half of d and d. +func jitter(d time.Duration) time.Duration { + return d/2 + rand.N(d/2+1) +} diff --git a/internal/server/reporter/reporter_test.go b/internal/server/reporter/reporter_test.go index f74ad8c..669e6e0 100644 --- a/internal/server/reporter/reporter_test.go +++ b/internal/server/reporter/reporter_test.go @@ -17,18 +17,27 @@ import ( "github.com/block/spectre/internal/server/client" "github.com/block/spectre/internal/server/reporter" "github.com/block/spectre/internal/server/storage" - "github.com/block/spectre/internal/stream" ) // updateProcedure is the protocol's path for streaming a mirror's state. const updateProcedure = "/spectre.v1.SpectreService/Update" -func TestReporterReconnectsWithQuarantine(t *testing.T) { - var attempts atomic.Int32 - service := server.NewService(storage.NewMemory(), time.Hour).Handler() +// mirror is a reporter running against a SPECTRE server. +type mirror struct { + spectre *client.Client + // streams counts the streams the reporter opened. + streams *atomic.Int32 + egress chan model.Quarantine +} + +// run runs a reporter against a server that lists states received within ttl +// and fails the first failures streams. +func run(t *testing.T, ttl time.Duration, failures int32, keepalive, maxAge time.Duration) mirror { + t.Helper() + streams := &atomic.Int32{} + service := server.NewService(storage.NewMemory(), ttl).Handler() httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - // The first stream fails, which the reporter recovers from. - if request.URL.Path == updateProcedure && attempts.Add(1) == 1 { + if request.URL.Path == updateProcedure && streams.Add(1) <= failures { // Reply without reading the stream, as a proxy in front of the server does. _ = http.NewResponseController(w).SetReadDeadline(time.Now()) //nolint:errcheck // HTTP/1 test server http.Error(w, "unavailable", http.StatusServiceUnavailable) @@ -37,45 +46,61 @@ func TestReporterReconnectsWithQuarantine(t *testing.T) { service.ServeHTTP(w, request) })) t.Cleanup(httpServer.Close) - ingress := make(chan model.Quarantine, 1) - egress := make(chan model.Quarantine, 1) config := reporter.Config{ - Client: client.Config{URL: httpServer.URL}, - Service: "users", - Environment: "production", - Revision: "abc123", - Stream: stream.Config{KeepaliveInterval: 10 * time.Millisecond, MaxAge: time.Hour}, + Client: client.Config{URL: httpServer.URL}, + Service: "users", + Environment: "production", + Revision: "abc123", + KeepaliveInterval: keepalive, + StreamMaxAge: maxAge, } spectre := client.New(config.Client) - mirror := reporter.New(config, spectre, reporter.Source{Direction: model.Ingress, Quarantined: ingress}, reporter.Source{Direction: model.Egress, Quarantined: egress}) + egress := make(chan model.Quarantine, 1) + ingress := make(chan model.Quarantine, 1) + reporting := reporter.New(config, spectre, reporter.Source{Direction: model.Ingress, Quarantined: ingress}, reporter.Source{Direction: model.Egress, Quarantined: egress}) ctx, cancel := context.WithCancel(t.Context()) t.Cleanup(cancel) - go mirror.Run(ctx) + go reporting.Run(ctx) + return mirror{spectre: spectre, streams: streams, egress: egress} +} + +// eventually fails the test unless done returns true within timeout. +func eventually(t *testing.T, timeout time.Duration, done func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !done() { + if time.Now().After(deadline) { + t.Fatal("timed out") + } + time.Sleep(5 * time.Millisecond) + } +} + +// states lists the states the server holds. +func (m mirror) states(t *testing.T) ([]model.State, []model.Divergence) { + t.Helper() + states, divergences, err := m.spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) + assert.NoError(t, err) + return states, divergences +} + +func TestReporterReconnectsWithQuarantine(t *testing.T) { + m := run(t, time.Hour, 1, 10*time.Millisecond, time.Hour) quarantine := model.Quarantine{Since: time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC), Reason: model.QuarantineReason{ Kind: model.QuarantineUnmatched, Traffic: Some(model.Traffic{Direction: model.Egress, Protocol: model.ProtocolHTTP, Route: "POST api.example/v1/charges"}), Detail: "no matching reference call", }} - egress <- quarantine + m.egress <- quarantine // Whichever stream the quarantine reaches, the next one begins with it. - deadline := time.After(10 * time.Second) - for { - _, divergences, err := spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) - assert.NoError(t, err) - if len(divergences) > 0 { - break - } - select { - case <-deadline: - t.Fatal("the reporter never reported the quarantine") - case <-time.After(10 * time.Millisecond): - } - } - assert.True(t, attempts.Load() > 1) + eventually(t, 10*time.Second, func() bool { + _, divergences := m.states(t) + return len(divergences) > 0 + }) + assert.True(t, m.streams.Load() > 1) - states, divergences, err := spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) - assert.NoError(t, err) + states, divergences := m.states(t) expected := []model.State{{ Service: "users", Environment: "production", @@ -88,3 +113,24 @@ func TestReporterReconnectsWithQuarantine(t *testing.T) { recorded := []model.Divergence{{Service: "users", Environment: "production", Revision: "abc123", Direction: model.Egress, Quarantine: quarantine}} assert.Equal(t, recorded, divergences) } + +func TestReporterKeepsQuietStreamListed(t *testing.T) { + m := run(t, 100*time.Millisecond, 0, 20*time.Millisecond, time.Hour) + eventually(t, 10*time.Second, func() bool { + states, _ := m.states(t) + return len(states) == 1 + }) + + // Keepalives outlast the server's TTL on one stream. + time.Sleep(500 * time.Millisecond) + states, _ := m.states(t) + assert.Equal(t, 1, len(states)) + assert.Equal(t, int32(1), m.streams.Load()) +} + +func TestReporterReplacesAgedStreamAtOnce(t *testing.T) { + m := run(t, time.Hour, 0, time.Hour, 20*time.Millisecond) + + // Failed streams would back off for at least 1.5s before a third. + eventually(t, time.Second, func() bool { return m.streams.Load() >= 3 }) +} diff --git a/internal/server/service.go b/internal/server/service.go index f508b9a..4dd171e 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -12,6 +12,7 @@ import ( "github.com/alecthomas/errors" . "github.com/alecthomas/types/optional" + "github.com/block/spectre/internal/httpserver/middleware/endearly" "github.com/block/spectre/internal/httpserver/middleware/health" "github.com/block/spectre/internal/httpserver/middleware/logging" "github.com/block/spectre/internal/logger" @@ -19,7 +20,6 @@ import ( "github.com/block/spectre/internal/netaddr" "github.com/block/spectre/internal/server/storage" "github.com/block/spectre/internal/server/wire" - "github.com/block/spectre/internal/stream" pb "github.com/block/spectre/spectrepb" "github.com/block/spectre/spectrepb/spectrepbconnect" ) @@ -42,7 +42,7 @@ func (s *Service) Handler() http.Handler { server := connect.NewServer() spectrepbconnect.RegisterSpectreServiceHandler(server, s) connecthttp.Mount(mux, server) - return stream.EndEarly(mux) + return endearly.New(mux) } // Update keeps the state a mirror streams. Every message stores the whole diff --git a/internal/stream/outchannel.go b/internal/stream/outchannel.go deleted file mode 100644 index e167347..0000000 --- a/internal/stream/outchannel.go +++ /dev/null @@ -1,155 +0,0 @@ -// Package stream keeps long-lived client streams open. An OutChannel keeps one -// stream from a client open through failures and sends it what producers queue. -// Every stream it opens begins with a resync of the client's whole state, so -// only events need queueing, and a stream may fail at any time without losing -// state. A quiet stream sends keepalives, which carry no message. EndEarly -// lets a server end such a stream before the client does. -package stream - -import ( - "context" - "math/rand/v2" - "time" - - "github.com/alecthomas/errors" - - "github.com/block/spectre/internal/logger" -) - -// Conn is one open client stream. -type Conn[M any] interface { - // Send sends message. It fails once the server has ended the stream. - Send(message M) error - // Keepalive shows the server that a quiet client is still connected. - Keepalive() error - // CloseAndReceive ends the stream and returns the error the server ended it with. - CloseAndReceive() error -} - -// Config configures how a channel keeps its stream open. -type Config struct { - KeepaliveInterval time.Duration `default:"30s" help:"How often to send a keepalive on a stream that is otherwise quiet."` - MaxAge time.Duration `name:"stream-max-age" default:"10m" help:"How long a stream lasts, with jitter, before it is replaced, so streams spread across servers."` -} - -// Validate checks that the intervals are positive. -func (c Config) Validate() error { - if c.KeepaliveInterval <= 0 || c.MaxAge <= 0 { - return errors.New("stream intervals must be positive") - } - return nil -} - -// Retries of a failed stream back off between these delays. -const ( - minRetryDelay = time.Second - maxRetryDelay = time.Minute -) - -// queueSize is how many messages wait for a stream before Send drops them. -const queueSize = 256 - -// OutChannel keeps one stream open and sends it the messages producers queue. -type OutChannel[M any] struct { - config Config - dial func(ctx context.Context) (Conn[M], error) - resync func() []M - queue chan M -} - -// NewOutChannel returns a channel that opens streams with dial. Each stream begins -// with the messages resync returns. -func NewOutChannel[M any](config Config, dial func(ctx context.Context) (Conn[M], error), resync func() []M) *OutChannel[M] { - return &OutChannel[M]{config: config, dial: dial, resync: resync, queue: make(chan M, queueSize)} -} - -// Send queues message for the stream without blocking. It drops the message -// when the queue is full. A queued message is lost if its stream fails. -func (c *OutChannel[M]) Send(message M) (queued bool) { - select { - case c.queue <- message: - return true - default: - return false - } -} - -// Run keeps a stream open until ctx ends, replacing it when it fails, with -// backoff, or when it reaches its age. Failures are only logged. -func (c *OutChannel[M]) Run(ctx context.Context) { - delay := minRetryDelay - for { - opened := time.Now() - err := c.stream(ctx) - if ctx.Err() != nil { - return - } - // Only consecutive quick failures back off further. - if err == nil || time.Since(opened) > maxRetryDelay { - delay = minRetryDelay - } - if err == nil { - continue - } - wait := jitter(delay) - logger.FromContext(ctx).WarnContext(ctx, "Lost a stream to the server", "error", err, "retry", wait) - select { - case <-ctx.Done(): - return - case <-time.After(wait): - } - delay = min(delay*2, maxRetryDelay) - } -} - -// stream sends on one stream until it fails or ctx ends. It returns nil once -// the stream reaches its age and the server accepts it closing. -func (c *OutChannel[M]) stream(ctx context.Context) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - conn, err := c.dial(ctx) - if err != nil { - return errors.Wrap(err, "open stream") - } - for _, message := range c.resync() { - if err := conn.Send(message); err != nil { - return failed(conn, err) - } - } - keepalive := time.NewTimer(c.config.KeepaliveInterval) - defer keepalive.Stop() - age := time.NewTimer(jitter(c.config.MaxAge)) - defer age.Stop() - for { - select { - case <-ctx.Done(): - cancel() - _ = conn.CloseAndReceive() //nolint:errcheck // Cancelling has already aborted the stream. - return errors.WithStack(ctx.Err()) - case <-age.C: - return errors.Wrap(conn.CloseAndReceive(), "close aged stream") - case <-keepalive.C: - err = conn.Keepalive() - case message := <-c.queue: - err = conn.Send(message) - } - if err != nil { - return failed(conn, err) - } - keepalive.Reset(c.config.KeepaliveInterval) - } -} - -// failed closes a stream whose send failed. A send fails once the server has -// ended the stream, and closing returns why it did. -func failed[M any](conn Conn[M], err error) error { - if closeErr := conn.CloseAndReceive(); closeErr != nil { - return errors.Wrap(closeErr, "stream ended") - } - return errors.Wrap(err, "send on stream") -} - -// jitter returns a random duration between half of d and d. -func jitter(d time.Duration) time.Duration { - return d/2 + rand.N(d/2+1) -} diff --git a/internal/stream/outchannel_test.go b/internal/stream/outchannel_test.go deleted file mode 100644 index 4f25a02..0000000 --- a/internal/stream/outchannel_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package stream_test - -import ( - "context" - "io" - "sync" - "testing" - "testing/synctest" - "time" - - "github.com/alecthomas/assert/v2" - "github.com/alecthomas/errors" - - "github.com/block/spectre/internal/stream" -) - -// conn records what a channel sends on one stream. -type conn struct { - mu sync.Mutex - sent []string - // Sends fail with io.EOF once this many messages are sent, unless it is zero. - limit int - closed bool -} - -func (c *conn) Send(message string) error { - c.mu.Lock() - defer c.mu.Unlock() - if c.limit > 0 && len(c.sent) >= c.limit { - return io.EOF - } - c.sent = append(c.sent, message) - return nil -} - -func (c *conn) Keepalive() error { - return c.Send("keepalive") -} - -func (c *conn) CloseAndReceive() error { - c.mu.Lock() - defer c.mu.Unlock() - c.closed = true - if c.limit > 0 { - return errors.New("server ended the stream") - } - return nil -} - -// sent is what a conn was sent and whether it was closed. -type sent struct { - Messages []string - Closed bool -} - -// dialer hands out a new conn for every stream a channel opens. -type dialer struct { - mu sync.Mutex - conns []*conn - limit []int -} - -func (d *dialer) dial(context.Context) (stream.Conn[string], error) { - d.mu.Lock() - defer d.mu.Unlock() - opened := &conn{} - if len(d.conns) < len(d.limit) { - opened.limit = d.limit[len(d.conns)] - } - d.conns = append(d.conns, opened) - return opened, nil -} - -func (d *dialer) streams() []sent { - d.mu.Lock() - defer d.mu.Unlock() - streams := make([]sent, 0, len(d.conns)) - for _, opened := range d.conns { - opened.mu.Lock() - streams = append(streams, sent{Messages: append([]string(nil), opened.sent...), Closed: opened.closed}) - opened.mu.Unlock() - } - return streams -} - -// run runs a channel until the test ends. -func run(t *testing.T, config stream.Config, d *dialer) *stream.OutChannel[string] { - t.Helper() - channel := stream.NewOutChannel(config, d.dial, func() []string { return []string{"state"} }) - ctx, cancel := context.WithCancel(t.Context()) - go channel.Run(ctx) - t.Cleanup(func() { - cancel() - synctest.Wait() - }) - return channel -} - -var config = stream.Config{KeepaliveInterval: 30 * time.Second, MaxAge: time.Hour} - -func TestOutChannelResyncsThenSendsAndKeepsAlive(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - d := &dialer{} - channel := run(t, config, d) - assert.True(t, channel.Send("event")) - synctest.Wait() - assert.Equal(t, []sent{{Messages: []string{"state", "event"}}}, d.streams()) - - // Only a quiet stream sends keepalives. - time.Sleep(20 * time.Second) - assert.True(t, channel.Send("later")) - time.Sleep(20 * time.Second) - synctest.Wait() - assert.Equal(t, []sent{{Messages: []string{"state", "event", "later"}}}, d.streams()) - time.Sleep(10 * time.Second) - synctest.Wait() - assert.Equal(t, []sent{{Messages: []string{"state", "event", "later", "keepalive"}}}, d.streams()) - }) -} - -func TestOutChannelReconnectsWithBackoff(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - // The first two streams end after the resync, the third lasts. - d := &dialer{limit: []int{1, 1}} - channel := run(t, config, d) - synctest.Wait() - assert.True(t, channel.Send("lost")) - synctest.Wait() - failed := sent{Messages: []string{"state"}, Closed: true} - assert.Equal(t, []sent{failed}, d.streams()) - - // Retries wait between half and all of a delay that doubles from 1s. - time.Sleep(time.Second) - synctest.Wait() - assert.Equal(t, []sent{failed, {Messages: []string{"state"}}}, d.streams()) - assert.True(t, channel.Send("lost again")) - time.Sleep(time.Second - time.Nanosecond) - synctest.Wait() - assert.Equal(t, 2, len(d.streams())) - time.Sleep(time.Second) - synctest.Wait() - assert.Equal(t, []sent{failed, failed, {Messages: []string{"state"}}}, d.streams()) - }) -} - -func TestOutChannelReplacesAgedStream(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - d := &dialer{} - run(t, stream.Config{KeepaliveInterval: time.Hour, MaxAge: 10 * time.Minute}, d) - synctest.Wait() - time.Sleep(5*time.Minute - time.Nanosecond) - synctest.Wait() - assert.Equal(t, []sent{{Messages: []string{"state"}}}, d.streams()) - - // A stream lasts between half and all of its age, and is replaced at once. - time.Sleep(5 * time.Minute) - synctest.Wait() - assert.Equal(t, []sent{{Messages: []string{"state"}, Closed: true}, {Messages: []string{"state"}}}, d.streams()) - }) -} - -func TestOutChannelDropsWhenQueueIsFull(t *testing.T) { - channel := stream.NewOutChannel(config, (&dialer{}).dial, func() []string { return nil }) - queued := 0 - for channel.Send("event") { - queued++ - } - assert.Equal(t, 256, queued) -} diff --git a/internal/supervisor/config.go b/internal/supervisor/config.go index a1e43fc..e197254 100644 --- a/internal/supervisor/config.go +++ b/internal/supervisor/config.go @@ -111,6 +111,9 @@ func (c Config) Validate() error { if c.MatchWindow <= 0 { return errors.New("match window must be positive") } + if err := c.Server.Validate(); err != nil { + return errors.WithStack(err) + } return errors.WithStack(c.Limits.Validate()) }