diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7667be9..7b1881b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,9 +64,8 @@ and `docker.io/blockossreleases/spectre`. ## SPECTRE server -`spectre serve` collects the heartbeats that mirrors send. Each heartbeat -carries the mirror's latest state: its candidate revision, SPECTRE version, -uptime, and quarantines. The server also records each quarantine as a +`spectre serve` collects the states that mirrors stream to it: each mirror's +candidate revision, SPECTRE version, uptime, and quarantines. The server also records each quarantine as a divergence that outlives the mirror run. It listens on `127.0.0.1:50070` by default and keeps everything in memory, so a restart loses it. To keep them in Redis instead, pass a Redis URL to `--storage`, such as `--storage=redis://localhost:6379/0`. For a Redis Cluster, use the @@ -86,14 +85,20 @@ pass the socket as `--server-socket` and the server's mesh address as `--server-url`. Every call dials the socket, and the mesh routes it by the URL's host, such as `--server-url=http://spectre.mesh --server-socket=@egress.sock`. -A mirror sends a heartbeat when it starts, every `--heartbeat-interval` (30s by -default), and as soon as it quarantines. A failed heartbeat is only logged. +A mirror sends everything to the server over one stream, which begins with its +whole state. It then sends each quarantine as it begins, and a keepalive every +`--server-keepalive-interval` (30s by default) when nothing else is sent. +Keep the interval below the idle timeout of any proxy in front of the server; +Caddy, for one, ends a request stream after 60s without data. When the stream +fails, the mirror logs it and reconnects with backoff, from 1s up to 1m. It also +replaces the stream after `--server-stream-max-age` (10m by default, with +jitter), so streams spread across server replicas. List the reporting mirrors and recorded divergences with `spectre status --server-url=URL`. Filter with `--service`, `--environment`, `--revision`, or `--node`, and print JSON with `--json`. A -mirror stays listed until it misses heartbeats for the server's `--state-ttl` -(90s by default), so keep that above the heartbeat interval. A mirror stays +mirror stays listed until it sends nothing for the server's `--state-ttl` +(90s by default), so keep that above the keepalive interval. A mirror stays quarantined until it restarts. The server keeps the latest divergence of each direction of a service diff --git a/cli_internal_test.go b/cli_internal_test.go index 490039e..a540193 100644 --- a/cli_internal_test.go +++ b/cli_internal_test.go @@ -32,7 +32,8 @@ func expectedReporting() reporter.Config { Service: "users", Environment: "production", Revision: "abc123", - HeartbeatInterval: 30 * time.Second, + KeepaliveInterval: 30 * time.Second, + StreamMaxAge: 10 * time.Minute, } } @@ -89,7 +90,8 @@ func TestSuperviseCommandReadsFlagsFromEnvironment(t *testing.T) { Service: "invoices", Environment: "production-us-west-2", Revision: "3f9c2a1", - HeartbeatInterval: 30 * time.Second, + KeepaliveInterval: 30 * time.Second, + StreamMaxAge: 10 * time.Minute, } assert.Equal(t, expected, command.Supervise.Supervisor.Server) } diff --git a/demo/README.md b/demo/README.md index f870dd1..5f7c61e 100644 --- a/demo/README.md +++ b/demo/README.md @@ -62,7 +62,7 @@ Base your image on the same pattern as `demo/Dockerfile`: (`--scripts-dir`), TypeScript schema (`--schema-dir`), and protobuf descriptors (`--descriptors-dir`). 3. Run the supervisor with your two commands, pointed at a SPECTRE server - (`spectre serve`) that it sends heartbeats to. + (`spectre serve`) that it streams its state to. The supervisor runs each service command through `/bin/sh -c` and exports two environment variables to it: diff --git a/demo/entrypoint.sh b/demo/entrypoint.sh index f8fef3f..1d510db 100755 --- a/demo/entrypoint.sh +++ b/demo/entrypoint.sh @@ -13,7 +13,7 @@ spectre-sample-forecasts \ --listen="unix:$sockets/forecasts.sock" \ --weather=/spectre/testdata/weather.json & -# Mirrors send heartbeats to the SPECTRE server, so run one alongside. +# Mirrors stream their states to the SPECTRE server, so run one alongside. spectre serve --listen=127.0.0.1:50070 & service="spectre-sample --listen=\$SPECTRE_LISTEN --forecasts=\$SPECTRE_EGRESS --data=/spectre/testdata/users.json --weather=/spectre/testdata/weather.json" diff --git a/internal/cli/commandline.go b/internal/cli/commandline.go index 1fba8cb..cc5760f 100644 --- a/internal/cli/commandline.go +++ b/internal/cli/commandline.go @@ -27,7 +27,7 @@ type CommandLine struct { Version kong.VersionFlag `env:"-" help:"Print the version and exit."` Supervise superviseCommand `cmd:"" help:"Supervise a candidate and a reference behind the embedded proxies, splitting resources between them."` - Serve serveCommand `cmd:"" help:"Run the SPECTRE server, which mirrors send their heartbeats to."` + Serve serveCommand `cmd:"" help:"Run the SPECTRE server, which mirrors stream their states to."` Gen genCommand `cmd:"" help:"Generate TypeScript schema declarations and the script API."` Status statusCommand `cmd:"" help:"List the mirrors reporting to the SPECTRE server and the divergences they reported, or clear divergences."` State stateCommand `cmd:"" help:"Describe a service's migration state."` diff --git a/internal/cli/status_internal_test.go b/internal/cli/status_internal_test.go index 44b0ddf..6bed0eb 100644 --- a/internal/cli/status_internal_test.go +++ b/internal/cli/status_internal_test.go @@ -54,8 +54,9 @@ func TestStatusCommands(t *testing.T) { Uptime: 12*time.Minute + 300*time.Millisecond, } for _, state := range []model.State{quarantined, healthy} { - assert.NoError(t, spectre.Update(t.Context(), state)) + assert.NoError(t, store.Put(t.Context(), state, time.Now())) } + assert.NoError(t, store.Record(t.Context(), model.Divergence{Service: "users", Environment: "production", Revision: "def456", Direction: model.Ingress, Quarantine: quarantined.Quarantines[model.Ingress]})) var text strings.Builder list := statusListCommand{} diff --git a/internal/httpserver/middleware/endearly/handler.go b/internal/httpserver/middleware/endearly/handler.go new file mode 100644 index 0000000..abe72ab --- /dev/null +++ b/internal/httpserver/middleware/endearly/handler.go @@ -0,0 +1,45 @@ +// Package endearly lets a server end a client stream before the client does. +package endearly + +import ( + "io" + "net/http" + "time" + + "github.com/alecthomas/errors" +) + +// New lets next end a client stream before the client does. Otherwise an +// HTTP/1 server reads the rest of the request before it replies. +func New(next http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + request.Body = newUnendedBody(request.Body, http.NewResponseController(writer)) + next.ServeHTTP(writer, request) + }) +} + +// unendedBody is a request body that the client may not have ended. +type unendedBody struct { + io.ReadCloser + controller *http.ResponseController + ended bool +} + +func newUnendedBody(body io.ReadCloser, controller *http.ResponseController) *unendedBody { + return &unendedBody{ReadCloser: body, controller: controller} +} + +func (b *unendedBody) Read(data []byte) (int, error) { + n, err := b.ReadCloser.Read(data) + b.ended = b.ended || errors.Is(err, io.EOF) + return n, err //nolint:wrapcheck // Readers must return io.EOF itself. +} + +// Close stops reading a body the client has not ended, so the server replies +// at once and then closes the connection, which tells the client. +func (b *unendedBody) Close() error { + if !b.ended { + _ = b.controller.SetReadDeadline(time.Now()) //nolint:errcheck // Without a deadline, closing only takes longer. + } + return errors.WithStack(b.ReadCloser.Close()) +} diff --git a/internal/model/state.go b/internal/model/state.go index bce22a3..9704ce8 100644 --- a/internal/model/state.go +++ b/internal/model/state.go @@ -1,9 +1,12 @@ package model -import "time" +import ( + "maps" + "time" +) // State is what the mirror of a service environment runs and holds, as its -// latest heartbeat reported it. +// latest update reported it. type State struct { Service string `json:"service"` Environment string `json:"environment"` @@ -16,3 +19,14 @@ type State struct { // Quarantines holds the quarantine of each direction whose proxy has one. Quarantines map[Direction]Quarantine `json:"quarantines,omitempty"` } + +func (State) update() {} + +// Quarantined returns the state with the quarantine of the proxy in direction. +func (s State) Quarantined(direction Direction, quarantine Quarantine) State { + quarantines := make(map[Direction]Quarantine, len(s.Quarantines)+1) + maps.Copy(quarantines, s.Quarantines) + quarantines[direction] = quarantine + s.Quarantines = quarantines + return s +} diff --git a/internal/model/update.go b/internal/model/update.go new file mode 100644 index 0000000..399c44b --- /dev/null +++ b/internal/model/update.go @@ -0,0 +1,15 @@ +package model + +// Update is one message of the stream a mirror sends the SPECTRE server. Every +// stream begins with the mirror's whole State. +// +//sumtype:decl +type Update interface{ update() } + +// QuarantineUpdate adds the quarantine of the proxy in Direction to the state. +type QuarantineUpdate struct { + Direction Direction + Quarantine Quarantine +} + +func (QuarantineUpdate) update() {} diff --git a/internal/server/client/client.go b/internal/server/client/client.go index 656f042..7dfc815 100644 --- a/internal/server/client/client.go +++ b/internal/server/client/client.go @@ -5,7 +5,6 @@ import ( "context" "net" "net/http" - "time" "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" @@ -17,9 +16,6 @@ import ( "github.com/block/spectre/spectrepb/spectrepbconnect" ) -// heartbeatTimeout bounds one heartbeat, so a slow server never delays the next. -const heartbeatTimeout = 10 * time.Second - // Config locates the SPECTRE server. type Config struct { URL string `name:"server-url" default:"${server_url}" placeholder:"URL" help:"Base URL of the SPECTRE server."` @@ -56,13 +52,14 @@ func New(config Config) *Client { return &Client{rpc: spectrepbconnect.NewSpectreServiceClient(connect.NewClient(connecthttp.NewTransport(httpClient, config.URL)))} } -// Update sends a mirror's latest state. It does not retry, because the -// next heartbeat replaces it. -func (c *Client) Update(ctx context.Context, state model.State) error { - ctx, cancel := context.WithTimeout(ctx, heartbeatTimeout) - defer cancel() - _, err := c.rpc.Update(ctx, &pb.UpdateRequest{State: wire.StateMessage(state)}) - return errors.Wrap(err, "send heartbeat") +// OpenUpdates opens a stream of a mirror's updates, which lasts until ctx ends +// or the stream is closed. +func (c *Client) OpenUpdates(ctx context.Context) (*Updates, error) { + stream, err := c.rpc.Update(ctx) + if err != nil { + return nil, errors.Wrap(err, "open update stream") + } + return newUpdates(stream), nil } // ListDivergences returns the states of the reporting mirrors and the @@ -82,3 +79,30 @@ func (c *Client) ClearDivergences(ctx context.Context, service, environment stri _, err := c.rpc.ClearDivergences(ctx, request) return errors.Wrap(err, "clear divergences") } + +// Updates streams changes to a mirror's state to the SPECTRE server. +type Updates struct { + stream spectrepbconnect.SpectreServiceUpdateClientStream +} + +func newUpdates(stream spectrepbconnect.SpectreServiceUpdateClientStream) *Updates { + return &Updates{stream: stream} +} + +// Send sends update. It fails once the server has ended the stream, and +// CloseAndReceive then returns why it did. +func (u *Updates) Send(update model.Update) error { + return errors.Wrap(u.stream.Send(wire.UpdateMessage(update)), "send update") +} + +// Keepalive shows the server that a quiet mirror is still connected. +func (u *Updates) Keepalive() error { + keepalive := &pb.UpdateRequest{Update: &pb.UpdateRequest_Keepalive{Keepalive: &pb.Keepalive{}}} + return errors.Wrap(u.stream.Send(keepalive), "send keepalive") +} + +// CloseAndReceive ends the stream and returns the error the server ended it with. +func (u *Updates) CloseAndReceive() error { + _, err := u.stream.CloseAndReceive() + return errors.Wrap(err, "close update stream") +} diff --git a/internal/server/config.go b/internal/server/config.go index 517b150..2e4537e 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -1,5 +1,5 @@ -// Package server is the SPECTRE server, which collects the heartbeats that -// mirrors send. It keeps their states and divergences in memory or in Redis. +// Package server is the SPECTRE server, which collects the states that mirrors +// stream. It keeps their states and divergences in memory or in Redis. package server import ( @@ -12,7 +12,7 @@ import ( type ServeConfig struct { Listen string `default:"127.0.0.1:50070" help:"Address for the SPECTRE server: host:port, or unix:."` Storage storage.Config `embed:""` - StateTTL time.Duration `default:"90s" help:"How long a mirror's state is listed after its last heartbeat. Keep it above the heartbeat interval."` + StateTTL time.Duration `default:"90s" help:"How long a mirror's state is listed after its last update. Keep it above the mirrors' keepalive interval."` } // ClearConfig names the service environment whose divergences to clear. diff --git a/internal/server/reporter/reporter.go b/internal/server/reporter/reporter.go index 9c9486b..44c2a4f 100644 --- a/internal/server/reporter/reporter.go +++ b/internal/server/reporter/reporter.go @@ -1,26 +1,48 @@ -// Package reporter sends a mirror's heartbeats to the SPECTRE server. +// Package reporter streams what a mirror sends the SPECTRE server over one +// stream. Every stream begins with the mirror's whole state, then sends each +// quarantine as it begins, so a stream may fail at any time without losing +// state. A quiet stream sends keepalives. A failed stream is replaced with +// backoff, and a stream is replaced when it ages, so streams spread across +// servers. package reporter import ( "context" - "maps" + "math/rand/v2" "time" + "github.com/alecthomas/errors" + "github.com/block/spectre/internal" "github.com/block/spectre/internal/logger" "github.com/block/spectre/internal/model" "github.com/block/spectre/internal/server/client" ) -// Config configures how a mirror reports its state to the SPECTRE server. +// Config configures how a mirror reports to the SPECTRE server. type Config struct { Client client.Config `embed:""` + KeepaliveInterval time.Duration `name:"server-keepalive-interval" default:"30s" help:"How often to send a keepalive on a stream to the SPECTRE server that is otherwise quiet."` + StreamMaxAge time.Duration `name:"server-stream-max-age" default:"10m" help:"How long a stream to the SPECTRE server lasts, with jitter, before it is replaced, so streams spread across servers."` Service string `required:"" help:"Service the mirror runs, reported to the SPECTRE server."` Environment string `required:"" help:"Environment the mirror runs in, such as production-us-west-2, reported to the SPECTRE server."` Revision string `required:"" help:"Commit of the candidate the mirror runs, reported to the SPECTRE server."` - HeartbeatInterval time.Duration `default:"30s" help:"How often the mirror sends its state to the SPECTRE server."` } +// Validate checks that the stream intervals are positive. +func (c Config) Validate() error { + if c.KeepaliveInterval <= 0 || c.StreamMaxAge <= 0 { + return errors.New("server keepalive interval and stream max age must be positive") + } + return nil +} + +// Retries of a failed stream back off between these delays. +const ( + minRetryDelay = time.Second + maxRetryDelay = time.Minute +) + // Source is a proxy whose quarantine a mirror reports. type Source struct { Direction model.Direction @@ -28,71 +50,123 @@ type Source struct { Quarantined <-chan model.Quarantine } -// Reporter sends a mirror's state to the SPECTRE server. +// Reporter streams a mirror's state to the SPECTRE server. type Reporter struct { config Config client *client.Client sources []Source + started time.Time + // quarantines belongs to the goroutine that runs streams. + quarantines map[model.Direction]model.Quarantine } // New reports the state of a mirror and its sources through server. func New(config Config, server *client.Client, sources ...Source) *Reporter { - return &Reporter{config: config, client: server, sources: sources} -} - -// sourceQuarantine is a quarantine of the source in direction. -type sourceQuarantine struct { - direction model.Direction - quarantine model.Quarantine + return &Reporter{config: config, client: server, sources: sources, started: time.Now(), quarantines: map[model.Direction]model.Quarantine{}} } -// Run sends a heartbeat at once, at every interval, and when a source -// quarantines, until ctx is cancelled. +// Run keeps a stream open until ctx ends, replacing it when it fails, with +// backoff, or when it ages. Failures are only logged. func (r *Reporter) Run(ctx context.Context) { - started := time.Now() - quarantined := make(chan sourceQuarantine) + // Each source sends once, so sending never blocks. + quarantines := make(chan model.QuarantineUpdate, len(r.sources)) for _, source := range r.sources { - go forward(ctx, source, quarantined) + go forward(ctx, source, quarantines) } - quarantines := map[model.Direction]model.Quarantine{} - ticker := time.NewTicker(r.config.HeartbeatInterval) - defer ticker.Stop() + delay := minRetryDelay for { - r.heartbeat(ctx, time.Since(started), quarantines) + opened := time.Now() + err := r.stream(ctx, quarantines) + if ctx.Err() != nil { + return + } + // Only consecutive quick failures back off further. + if err == nil || time.Since(opened) > maxRetryDelay { + delay = minRetryDelay + } + if err == nil { + continue + } + wait := jitter(delay) + logger.FromContext(ctx).WarnContext(ctx, "Lost the stream to the SPECTRE server", "error", err, "retry", wait) select { case <-ctx.Done(): return - case <-ticker.C: - case status := <-quarantined: - quarantines[status.direction] = status.quarantine + case <-time.After(wait): } + delay = min(delay*2, maxRetryDelay) } } -// heartbeat sends the mirror's state. A failure is only logged, so the -// SPECTRE server never blocks the mirror. -func (r *Reporter) heartbeat(ctx context.Context, uptime time.Duration, quarantines map[model.Direction]model.Quarantine) { - state := model.State{ +// stream sends on one stream until it fails or ctx ends. It returns nil once +// the stream ages and the server accepts it closing. +func (r *Reporter) stream(ctx context.Context, quarantines <-chan model.QuarantineUpdate) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + updates, err := r.client.OpenUpdates(ctx) + if err != nil { + return errors.Wrap(err, "open stream") + } + if err := updates.Send(r.state()); err != nil { + return failed(updates, err) + } + keepalive := time.NewTimer(r.config.KeepaliveInterval) + defer keepalive.Stop() + age := time.NewTimer(jitter(r.config.StreamMaxAge)) + defer age.Stop() + for { + select { + case <-ctx.Done(): + cancel() + _ = updates.CloseAndReceive() //nolint:errcheck // Cancelling has already aborted the stream. + return errors.WithStack(ctx.Err()) + case <-age.C: + return errors.Wrap(updates.CloseAndReceive(), "close aged stream") + case <-keepalive.C: + err = updates.Keepalive() + case update := <-quarantines: + // Recorded first, so the next stream's state holds it if this send fails. + r.quarantines[update.Direction] = update.Quarantine + err = updates.Send(update) + } + if err != nil { + return failed(updates, err) + } + keepalive.Reset(r.config.KeepaliveInterval) + } +} + +// state returns the mirror's whole state, to send at once. +func (r *Reporter) state() model.State { + return model.State{ Service: r.config.Service, Environment: r.config.Environment, Revision: r.config.Revision, SpectreVersion: internal.Version, - Uptime: uptime, - Quarantines: maps.Clone(quarantines), - } - if err := r.client.Update(ctx, state); err != nil && ctx.Err() == nil { - logger.FromContext(ctx).WarnContext(ctx, "Failed to send heartbeat to the SPECTRE server", "error", err) + Uptime: time.Since(r.started), + Quarantines: r.quarantines, } } -// forward passes the source's quarantine on to quarantined. -func forward(ctx context.Context, source Source, quarantined chan<- sourceQuarantine) { +// forward sends the source's quarantine to quarantines once it begins. +func forward(ctx context.Context, source Source, quarantines chan<- model.QuarantineUpdate) { select { - case status := <-source.Quarantined: - select { - case quarantined <- sourceQuarantine{direction: source.Direction, quarantine: status}: - case <-ctx.Done(): - } + case quarantine := <-source.Quarantined: + quarantines <- model.QuarantineUpdate{Direction: source.Direction, Quarantine: quarantine} case <-ctx.Done(): } } + +// failed closes a stream whose send failed. A send fails once the server has +// ended the stream, and closing returns why it did. +func failed(updates *client.Updates, err error) error { + if closeErr := updates.CloseAndReceive(); closeErr != nil { + return errors.Wrap(closeErr, "stream ended") + } + return errors.Wrap(err, "send on stream") +} + +// jitter returns a random duration between half of d and d. +func jitter(d time.Duration) time.Duration { + return d/2 + rand.N(d/2+1) +} diff --git a/internal/server/reporter/reporter_test.go b/internal/server/reporter/reporter_test.go index 6d7b620..669e6e0 100644 --- a/internal/server/reporter/reporter_test.go +++ b/internal/server/reporter/reporter_test.go @@ -19,54 +19,88 @@ import ( "github.com/block/spectre/internal/server/storage" ) -// updateProcedure is the protocol's path for sending a heartbeat. +// updateProcedure is the protocol's path for streaming a mirror's state. const updateProcedure = "/spectre.v1.SpectreService/Update" -func TestReporterSendsHeartbeatOnQuarantine(t *testing.T) { - heartbeats := make(chan int, 8) - var attempts atomic.Int32 - service := server.NewService(storage.NewMemory(), time.Hour).Handler() +// mirror is a reporter running against a SPECTRE server. +type mirror struct { + spectre *client.Client + // streams counts the streams the reporter opened. + streams *atomic.Int32 + egress chan model.Quarantine +} + +// run runs a reporter against a server that lists states received within ttl +// and fails the first failures streams. +func run(t *testing.T, ttl time.Duration, failures int32, keepalive, maxAge time.Duration) mirror { + t.Helper() + streams := &atomic.Int32{} + service := server.NewService(storage.NewMemory(), ttl).Handler() httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - if request.URL.Path != updateProcedure { - service.ServeHTTP(w, request) - return - } - attempt := int(attempts.Add(1)) - // The first heartbeat fails, which the next one recovers from. - if attempt == 1 { + if request.URL.Path == updateProcedure && streams.Add(1) <= failures { + // Reply without reading the stream, as a proxy in front of the server does. + _ = http.NewResponseController(w).SetReadDeadline(time.Now()) //nolint:errcheck // HTTP/1 test server http.Error(w, "unavailable", http.StatusServiceUnavailable) - } else { - service.ServeHTTP(w, request) + return } - heartbeats <- attempt + service.ServeHTTP(w, request) })) t.Cleanup(httpServer.Close) - ingress := make(chan model.Quarantine, 1) - egress := make(chan model.Quarantine, 1) config := reporter.Config{ Client: client.Config{URL: httpServer.URL}, Service: "users", Environment: "production", Revision: "abc123", - HeartbeatInterval: time.Hour, + KeepaliveInterval: keepalive, + StreamMaxAge: maxAge, } spectre := client.New(config.Client) - mirror := reporter.New(config, spectre, reporter.Source{Direction: model.Ingress, Quarantined: ingress}, reporter.Source{Direction: model.Egress, Quarantined: egress}) + egress := make(chan model.Quarantine, 1) + ingress := make(chan model.Quarantine, 1) + reporting := reporter.New(config, spectre, reporter.Source{Direction: model.Ingress, Quarantined: ingress}, reporter.Source{Direction: model.Egress, Quarantined: egress}) ctx, cancel := context.WithCancel(t.Context()) t.Cleanup(cancel) - go mirror.Run(ctx) - assert.Equal(t, 1, <-heartbeats) + go reporting.Run(ctx) + return mirror{spectre: spectre, streams: streams, egress: egress} +} +// eventually fails the test unless done returns true within timeout. +func eventually(t *testing.T, timeout time.Duration, done func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for !done() { + if time.Now().After(deadline) { + t.Fatal("timed out") + } + time.Sleep(5 * time.Millisecond) + } +} + +// states lists the states the server holds. +func (m mirror) states(t *testing.T) ([]model.State, []model.Divergence) { + t.Helper() + states, divergences, err := m.spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) + assert.NoError(t, err) + return states, divergences +} + +func TestReporterReconnectsWithQuarantine(t *testing.T) { + m := run(t, time.Hour, 1, 10*time.Millisecond, time.Hour) quarantine := model.Quarantine{Since: time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC), Reason: model.QuarantineReason{ Kind: model.QuarantineUnmatched, Traffic: Some(model.Traffic{Direction: model.Egress, Protocol: model.ProtocolHTTP, Route: "POST api.example/v1/charges"}), Detail: "no matching reference call", }} - egress <- quarantine - assert.Equal(t, 2, <-heartbeats) + m.egress <- quarantine - states, divergences, err := spectre.ListDivergences(t.Context(), model.DivergenceFilter{}) - assert.NoError(t, err) + // Whichever stream the quarantine reaches, the next one begins with it. + eventually(t, 10*time.Second, func() bool { + _, divergences := m.states(t) + return len(divergences) > 0 + }) + assert.True(t, m.streams.Load() > 1) + + states, divergences := m.states(t) expected := []model.State{{ Service: "users", Environment: "production", @@ -79,3 +113,24 @@ func TestReporterSendsHeartbeatOnQuarantine(t *testing.T) { recorded := []model.Divergence{{Service: "users", Environment: "production", Revision: "abc123", Direction: model.Egress, Quarantine: quarantine}} assert.Equal(t, recorded, divergences) } + +func TestReporterKeepsQuietStreamListed(t *testing.T) { + m := run(t, 100*time.Millisecond, 0, 20*time.Millisecond, time.Hour) + eventually(t, 10*time.Second, func() bool { + states, _ := m.states(t) + return len(states) == 1 + }) + + // Keepalives outlast the server's TTL on one stream. + time.Sleep(500 * time.Millisecond) + states, _ := m.states(t) + assert.Equal(t, 1, len(states)) + assert.Equal(t, int32(1), m.streams.Load()) +} + +func TestReporterReplacesAgedStreamAtOnce(t *testing.T) { + m := run(t, time.Hour, 0, time.Hour, 20*time.Millisecond) + + // Failed streams would back off for at least 1.5s before a third. + eventually(t, time.Second, func() bool { return m.streams.Load() >= 3 }) +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 6fafd44..5f30288 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,6 +1,7 @@ package server_test import ( + "io" "net/http/httptest" "testing" "time" @@ -8,6 +9,7 @@ import ( "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" "github.com/alecthomas/assert/v2" + "github.com/alecthomas/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -36,6 +38,54 @@ func state(environment string, uptime time.Duration, quarantines ...*pb.Quaranti } } +// update streams requests to the server, then closes the stream and returns +// the error the server ended it with. +func update(t *testing.T, rpc spectrepbconnect.SpectreServiceClient, requests ...*pb.UpdateRequest) error { + t.Helper() + stream, err := rpc.Update(t.Context()) + assert.NoError(t, err) + for _, request := range requests { + // A send fails with io.EOF once the server has ended the stream. + if err := stream.Send(request); err != nil && !errors.Is(err, io.EOF) { + return err + } + } + _, err = stream.CloseAndReceive() + return err +} + +func whole(state *pb.State) *pb.UpdateRequest { + return &pb.UpdateRequest{Update: &pb.UpdateRequest_State{State: state}} +} + +func quarantined(quarantine *pb.Quarantine) *pb.UpdateRequest { + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Quarantine{Quarantine: quarantine}} +} + +func keepalive() *pb.UpdateRequest { + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Keepalive{Keepalive: &pb.Keepalive{}}} +} + +// list returns what the server lists for request without uptimes, which it +// returns separately because the server advances them. +func list(t *testing.T, rpc spectrepbconnect.SpectreServiceClient, request *pb.ListDivergencesRequest) (listed *pb.ListDivergencesResponse, uptimes []time.Duration) { + t.Helper() + response, err := rpc.ListDivergences(t.Context(), request) + assert.NoError(t, err) + for _, listedState := range response.GetStates() { + uptimes = append(uptimes, listedState.GetUptime().AsDuration()) + listedState.Uptime = nil + } + return response, uptimes +} + +// withoutUptime returns a copy of state without its uptime, as list returns it. +func withoutUptime(state *pb.State) *pb.State { + stripped := proto.CloneOf(state) + stripped.Uptime = nil + return stripped +} + func TestUpdateKeepsLatestState(t *testing.T) { rpc := newRPC(t) since := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) @@ -47,67 +97,60 @@ func TestUpdateKeepsLatestState(t *testing.T) { Node: proto.String("users"), Differences: []string{"body.name"}, } - latest := state("production", 2*time.Minute, quarantine) - divergence := &pb.Divergence{Service: "users", Environment: "production", Revision: "abc123", Quarantine: quarantine} staging := state("staging", time.Minute) - for _, sent := range []*pb.State{state("production", time.Minute), staging, latest} { - _, err := rpc.Update(t.Context(), &pb.UpdateRequest{State: sent}) - assert.NoError(t, err) - } + assert.NoError(t, update(t, rpc, whole(state("production", time.Hour)), keepalive(), quarantined(quarantine), keepalive())) + assert.NoError(t, update(t, rpc, whole(staging))) - all := &pb.ListDivergencesResponse{States: []*pb.State{latest, staging}, Divergences: []*pb.Divergence{divergence}} + production := withoutUptime(state("production", time.Hour, quarantine)) + divergence := &pb.Divergence{Service: "users", Environment: "production", Revision: "abc123", Quarantine: quarantine} tests := []struct { name string request *pb.ListDivergencesRequest expected *pb.ListDivergencesResponse }{ - {name: "All", request: &pb.ListDivergencesRequest{}, expected: all}, - {name: "Environment", request: &pb.ListDivergencesRequest{Environment: proto.String("staging")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{staging}}}, - {name: "Node", request: &pb.ListDivergencesRequest{Node: proto.String("users")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{latest}, Divergences: []*pb.Divergence{divergence}}}, + {name: "All", request: &pb.ListDivergencesRequest{}, expected: &pb.ListDivergencesResponse{States: []*pb.State{production, withoutUptime(staging)}, Divergences: []*pb.Divergence{divergence}}}, + {name: "Environment", request: &pb.ListDivergencesRequest{Environment: proto.String("staging")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{withoutUptime(staging)}}}, + {name: "Node", request: &pb.ListDivergencesRequest{Node: proto.String("users")}, expected: &pb.ListDivergencesResponse{States: []*pb.State{production}, Divergences: []*pb.Divergence{divergence}}}, {name: "Revision", request: &pb.ListDivergencesRequest{Revision: proto.String("def456")}, expected: &pb.ListDivergencesResponse{}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - listed, err := rpc.ListDivergences(t.Context(), test.request) - assert.NoError(t, err) + listed, _ := list(t, rpc, test.request) assert.True(t, proto.Equal(test.expected, listed), "%v", listed) }) } + + // Later updates advance the uptime the mirror sent with its whole state. + _, uptimes := list(t, rpc, &pb.ListDivergencesRequest{}) + assert.True(t, uptimes[0] > time.Hour && uptimes[0] < time.Hour+time.Minute, "%v", uptimes[0]) + assert.Equal(t, time.Minute, uptimes[1]) } func TestDivergencesOutliveRestartUntilCleared(t *testing.T) { rpc := newRPC(t) quarantine := &pb.Quarantine{Direction: pb.Direction_DIRECTION_EGRESS, Since: timestamppb.Now(), Kind: pb.Quarantine_KIND_UNMATCHED} - quarantined := state("production", time.Minute, quarantine) redeployed := state("production", time.Second) redeployed.Revision = "def456" - heartbeat := func(sent *pb.State) { - _, err := rpc.Update(t.Context(), &pb.UpdateRequest{State: sent}) - assert.NoError(t, err) - } - list := func() *pb.ListDivergencesResponse { - listed, err := rpc.ListDivergences(t.Context(), &pb.ListDivergencesRequest{}) - assert.NoError(t, err) - return listed - } - heartbeat(quarantined) - heartbeat(redeployed) + assert.NoError(t, update(t, rpc, whole(state("production", time.Minute)), quarantined(quarantine))) + assert.NoError(t, update(t, rpc, whole(redeployed))) divergence := &pb.Divergence{Service: "users", Environment: "production", Revision: "abc123", Quarantine: quarantine} - expected := &pb.ListDivergencesResponse{States: []*pb.State{redeployed}, Divergences: []*pb.Divergence{divergence}} - assert.True(t, proto.Equal(expected, list()), "%v", list()) + expected := &pb.ListDivergencesResponse{States: []*pb.State{withoutUptime(redeployed)}, Divergences: []*pb.Divergence{divergence}} + listed, _ := list(t, rpc, &pb.ListDivergencesRequest{}) + assert.True(t, proto.Equal(expected, listed), "%v", listed) _, err := rpc.ClearDivergences(t.Context(), &pb.ClearDivergencesRequest{Service: "users", Environment: "production"}) assert.NoError(t, err) - // A heartbeat that was in flight still carries the cleared quarantine. - heartbeat(quarantined) - heartbeat(redeployed) - assert.True(t, proto.Equal(&pb.ListDivergencesResponse{States: []*pb.State{redeployed}}, list()), "%v", list()) + // A mirror still running reports the cleared quarantine again. + assert.NoError(t, update(t, rpc, whole(state("production", time.Minute, quarantine)))) + assert.NoError(t, update(t, rpc, whole(redeployed))) + listed, _ = list(t, rpc, &pb.ListDivergencesRequest{}) + assert.True(t, proto.Equal(&pb.ListDivergencesResponse{States: []*pb.State{withoutUptime(redeployed)}}, listed), "%v", listed) _, err = rpc.ClearDivergences(t.Context(), &pb.ClearDivergencesRequest{Service: "users"}) assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) } -func TestUpdateRejectsInvalidState(t *testing.T) { +func TestUpdateRejectsInvalidStream(t *testing.T) { rpc := newRPC(t) quarantine := func(direction pb.Direction) *pb.Quarantine { return &pb.Quarantine{Direction: direction, Since: timestamppb.Now(), Kind: pb.Quarantine_KIND_UNMATCHED} @@ -121,21 +164,42 @@ func TestUpdateRejectsInvalidState(t *testing.T) { withoutTrafficDirection := state("production", time.Minute, quarantine(pb.Direction_DIRECTION_INGRESS)) withoutTrafficDirection.Quarantines[0].Traffic = &pb.Traffic{Protocol: "http"} tests := []struct { - name string - state *pb.State + name string + requests []*pb.UpdateRequest }{ - {name: "WithoutState"}, - {name: "WithoutRevision", state: withoutCommit}, - {name: "WithoutUptime", state: withoutUptime}, - {name: "UnspecifiedQuarantineDirection", state: state("production", time.Minute, quarantine(pb.Direction_DIRECTION_UNSPECIFIED))}, - {name: "RepeatedDirection", state: state("production", time.Minute, quarantine(pb.Direction_DIRECTION_INGRESS), quarantine(pb.Direction_DIRECTION_INGRESS))}, - {name: "UnspecifiedKind", state: withoutKind}, - {name: "UnspecifiedTrafficDirection", state: withoutTrafficDirection}, + {name: "Empty", requests: []*pb.UpdateRequest{{}}}, + {name: "WithoutState", requests: []*pb.UpdateRequest{whole(nil)}}, + {name: "BeginsWithKeepalive", requests: []*pb.UpdateRequest{keepalive()}}, + {name: "BeginsWithQuarantine", requests: []*pb.UpdateRequest{quarantined(quarantine(pb.Direction_DIRECTION_INGRESS))}}, + {name: "WithoutRevision", requests: []*pb.UpdateRequest{whole(withoutCommit)}}, + {name: "WithoutUptime", requests: []*pb.UpdateRequest{whole(withoutUptime)}}, + {name: "UnspecifiedQuarantineDirection", requests: []*pb.UpdateRequest{whole(state("production", time.Minute, quarantine(pb.Direction_DIRECTION_UNSPECIFIED)))}}, + {name: "RepeatedDirection", requests: []*pb.UpdateRequest{whole(state("production", time.Minute, quarantine(pb.Direction_DIRECTION_INGRESS), quarantine(pb.Direction_DIRECTION_INGRESS)))}}, + {name: "UnspecifiedKind", requests: []*pb.UpdateRequest{whole(withoutKind)}}, + {name: "UnspecifiedTrafficDirection", requests: []*pb.UpdateRequest{whole(withoutTrafficDirection)}}, + {name: "InvalidQuarantine", requests: []*pb.UpdateRequest{whole(state("production", time.Minute)), quarantined(quarantine(pb.Direction_DIRECTION_UNSPECIFIED))}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := rpc.Update(t.Context(), &pb.UpdateRequest{State: test.state}) + err := update(t, rpc, test.requests...) assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) }) } } + +func TestUpdateEndsRejectedStreamWhileMirrorSends(t *testing.T) { + rpc := newRPC(t) + stream, err := rpc.Update(t.Context()) + assert.NoError(t, err) + assert.NoError(t, stream.Send(whole(state("production", time.Minute)))) + assert.NoError(t, stream.Send(quarantined(&pb.Quarantine{Since: timestamppb.Now(), Kind: pb.Quarantine_KIND_UNMATCHED}))) + // A mirror only learns that the server ended its stream when a send fails. + deadline := time.Now().Add(10 * time.Second) + for err == nil && time.Now().Before(deadline) { + err = stream.Send(keepalive()) + time.Sleep(time.Millisecond) + } + assert.IsError(t, err, io.EOF) + _, err = stream.CloseAndReceive() + assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) +} diff --git a/internal/server/service.go b/internal/server/service.go index 6e06fbb..4dd171e 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -2,6 +2,7 @@ package server import ( "context" + "io" "net" "net/http" "time" @@ -9,7 +10,9 @@ import ( "connectrpc.com/connect/v2" "connectrpc.com/connect/v2/connecthttp" "github.com/alecthomas/errors" + . "github.com/alecthomas/types/optional" + "github.com/block/spectre/internal/httpserver/middleware/endearly" "github.com/block/spectre/internal/httpserver/middleware/health" "github.com/block/spectre/internal/httpserver/middleware/logging" "github.com/block/spectre/internal/logger" @@ -39,19 +42,48 @@ func (s *Service) Handler() http.Handler { server := connect.NewServer() spectrepbconnect.RegisterSpectreServiceHandler(server, s) connecthttp.Mount(mux, server) - return mux + return endearly.New(mux) } -// Update records the latest state of a mirror, and the divergence of each -// of its quarantines. -func (s *Service) Update(ctx context.Context, request *pb.UpdateRequest) (*pb.UpdateResponse, error) { - state, err := wire.StateFromMessage(request.GetState()) - if err != nil { - return nil, connect.NewError(connect.CodeInvalidArgument, err.Error()).WithCause(err) - } - if err := s.store.Put(ctx, state, time.Now()); err != nil { - return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) +// Update keeps the state a mirror streams. Every message stores the whole +// state, which keeps it listed, and records the divergence of each quarantine. +func (s *Service) Update(ctx context.Context, updates spectrepbconnect.SpectreServiceUpdateServerStream) (*pb.UpdateResponse, error) { + var state Option[model.State] + var measured time.Time + for { + message, err := updates.Receive() + if errors.Is(err, io.EOF) { + return &pb.UpdateResponse{}, nil + } + if err != nil { + return nil, errors.WithStack(err) + } + current, err := nextState(state, message) + if err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err.Error()).WithCause(err) + } + now := time.Now() + if message.GetState() != nil { + measured = now + } + state = Some(current) + // The mirror measured its uptime when it sent the whole state. + stored := current + stored.Uptime += now.Sub(measured) + if err := s.store.Put(ctx, stored, now); err != nil { + return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) + } + if message.GetKeepalive() != nil { + continue + } + if err := s.recordQuarantines(ctx, current); err != nil { + return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) + } } +} + +// recordQuarantines records the divergence of each quarantine state holds. +func (s *Service) recordQuarantines(ctx context.Context, state model.State) error { for direction, quarantine := range state.Quarantines { divergence := model.Divergence{ Service: state.Service, @@ -61,14 +93,14 @@ func (s *Service) Update(ctx context.Context, request *pb.UpdateRequest) (*pb.Up Quarantine: quarantine, } if err := s.store.Record(ctx, divergence); err != nil { - return nil, connect.NewError(connect.CodeUnavailable, err.Error()).WithCause(err) + return errors.WithStack(err) } } - return &pb.UpdateResponse{}, nil + return nil } -// ListDivergences returns the states of the mirrors that sent a heartbeat -// within the TTL, and every recorded divergence. +// ListDivergences returns the states of the mirrors that sent an update within +// the TTL, and every recorded divergence. func (s *Service) ListDivergences(ctx context.Context, request *pb.ListDivergencesRequest) (*pb.ListDivergencesResponse, error) { states, err := s.store.States(ctx, time.Now().Add(-s.ttl)) if err != nil { @@ -129,3 +161,29 @@ func Serve(ctx context.Context, config ServeConfig) error { } return nil } + +// nextState applies a streamed update to the state a stream has built. A +// stream must begin with the whole state, and a keepalive leaves it as it is. +func nextState(state Option[model.State], message *pb.UpdateRequest) (model.State, error) { + current, started := state.Get() + if message.GetKeepalive() != nil { + if !started { + return model.State{}, errors.New("an update stream must begin with the mirror's state") + } + return current, nil + } + update, err := wire.UpdateFromMessage(message) + if err != nil { + return model.State{}, errors.WithStack(err) + } + switch update := update.(type) { + case model.State: + return update, nil + case model.QuarantineUpdate: + if !started { + return model.State{}, errors.New("an update stream must begin with the mirror's state") + } + return current.Quarantined(update.Direction, update.Quarantine), nil + } + return current, nil +} diff --git a/internal/server/storage/storage_test.go b/internal/server/storage/storage_test.go index ec83fc8..f283256 100644 --- a/internal/server/storage/storage_test.go +++ b/internal/server/storage/storage_test.go @@ -252,10 +252,10 @@ func testKeepsLatestQuarantine(t *testing.T, store storage.Store) { for _, reported := range []model.Divergence{ divergence("production", model.Ingress, first), latest, - // A heartbeat from before the mirror restarted and quarantined again. + // An update from before the mirror restarted and quarantined again. divergence("production", model.Ingress, first.Add(-time.Minute)), egress, - // Every heartbeat repeats the quarantine. + // Every update repeats the quarantine. egress, } { assert.NoError(t, store.Record(t.Context(), reported)) diff --git a/internal/server/wire/wire.go b/internal/server/wire/wire.go index 99b0a76..ca710b0 100644 --- a/internal/server/wire/wire.go +++ b/internal/server/wire/wire.go @@ -25,7 +25,7 @@ func StateMessage(state model.State) *pb.State { // Ingress first, so a state always encodes the same way. for _, direction := range []model.Direction{model.Ingress, model.Egress} { if quarantine, ok := state.Quarantines[direction]; ok { - message.Quarantines = append(message.Quarantines, quarantineMessage(direction, quarantine)) + message.Quarantines = append(message.Quarantines, QuarantineMessage(direction, quarantine)) } } return message @@ -52,29 +52,53 @@ func StateFromMessage(message *pb.State) (model.State, error) { state.Quarantines = make(map[model.Direction]model.Quarantine, len(quarantines)) } for _, quarantine := range quarantines { - direction, err := directionFromMessage(quarantine.GetDirection()) + direction, restored, err := QuarantineFromMessage(quarantine) if err != nil { - return model.State{}, errors.Wrap(err, "quarantine") + return model.State{}, err } if _, ok := state.Quarantines[direction]; ok { return model.State{}, errors.Errorf("state holds more than one %s quarantine", direction) } - restored, err := quarantineFromMessage(quarantine) - if err != nil { - return model.State{}, errors.Wrap(err, "quarantine") - } state.Quarantines[direction] = restored } return state, nil } +// UpdateMessage returns the message that streams update. +func UpdateMessage(update model.Update) *pb.UpdateRequest { + switch update := update.(type) { + case model.State: + return &pb.UpdateRequest{Update: &pb.UpdateRequest_State{State: StateMessage(update)}} + case model.QuarantineUpdate: + return &pb.UpdateRequest{Update: &pb.UpdateRequest_Quarantine{Quarantine: QuarantineMessage(update.Direction, update.Quarantine)}} + } + return &pb.UpdateRequest{} +} + +// UpdateFromMessage restores a streamed update. A keepalive holds none. +func UpdateFromMessage(message *pb.UpdateRequest) (model.Update, error) { + switch update := message.GetUpdate().(type) { + case *pb.UpdateRequest_State: + return StateFromMessage(update.State) + case *pb.UpdateRequest_Quarantine: + direction, quarantine, err := QuarantineFromMessage(update.Quarantine) + if err != nil { + return nil, err + } + return model.QuarantineUpdate{Direction: direction, Quarantine: quarantine}, nil + case *pb.UpdateRequest_Keepalive: + return nil, errors.New("a keepalive holds no update") + } + return nil, errors.New("update holds nothing") +} + // DivergenceMessage returns the message that reports divergence. func DivergenceMessage(divergence model.Divergence) *pb.Divergence { return &pb.Divergence{ Service: divergence.Service, Environment: divergence.Environment, Revision: divergence.Revision, - Quarantine: quarantineMessage(divergence.Direction, divergence.Quarantine), + Quarantine: QuarantineMessage(divergence.Direction, divergence.Quarantine), } } @@ -87,13 +111,9 @@ func DivergenceFromMessage(message *pb.Divergence) (model.Divergence, error) { if message.GetQuarantine() == nil { return model.Divergence{}, errors.New("divergence needs a quarantine") } - direction, err := directionFromMessage(message.GetQuarantine().GetDirection()) - if err != nil { - return model.Divergence{}, errors.Wrap(err, "quarantine") - } - quarantine, err := quarantineFromMessage(message.GetQuarantine()) + direction, quarantine, err := QuarantineFromMessage(message.GetQuarantine()) if err != nil { - return model.Divergence{}, errors.Wrap(err, "quarantine") + return model.Divergence{}, err } return model.Divergence{ Service: message.GetService(), @@ -168,7 +188,9 @@ func FilterFromMessage(request *pb.ListDivergencesRequest) model.DivergenceFilte } } -func quarantineMessage(direction model.Direction, status model.Quarantine) *pb.Quarantine { +// QuarantineMessage returns the message that reports the quarantine of the +// proxy in direction. +func QuarantineMessage(direction model.Direction, status model.Quarantine) *pb.Quarantine { reason := status.Reason message := &pb.Quarantine{ Direction: directionMessage(direction), @@ -188,6 +210,20 @@ func quarantineMessage(direction model.Direction, status model.Quarantine) *pb.Q return message } +// QuarantineFromMessage restores a reported quarantine and the direction of +// the proxy that holds it. +func QuarantineFromMessage(message *pb.Quarantine) (model.Direction, model.Quarantine, error) { + direction, err := directionFromMessage(message.GetDirection()) + if err != nil { + return "", model.Quarantine{}, errors.Wrap(err, "quarantine") + } + quarantine, err := quarantineFromMessage(message) + if err != nil { + return "", model.Quarantine{}, errors.Wrap(err, "quarantine") + } + return direction, quarantine, nil +} + // quarantineFromMessage restores a reported quarantine, which must say when it began. func quarantineFromMessage(message *pb.Quarantine) (model.Quarantine, error) { if message.GetSince() == nil { diff --git a/internal/supervisor/config.go b/internal/supervisor/config.go index a1e43fc..e197254 100644 --- a/internal/supervisor/config.go +++ b/internal/supervisor/config.go @@ -111,6 +111,9 @@ func (c Config) Validate() error { if c.MatchWindow <= 0 { return errors.New("match window must be positive") } + if err := c.Server.Validate(); err != nil { + return errors.WithStack(err) + } return errors.WithStack(c.Limits.Validate()) } diff --git a/internal/supervisor/supervisor.go b/internal/supervisor/supervisor.go index 9bab3a5..48e9c32 100644 --- a/internal/supervisor/supervisor.go +++ b/internal/supervisor/supervisor.go @@ -211,7 +211,7 @@ func (s *Supervisor) startAll(ctx context.Context) error { return nil } -// startReporter sends the mirror's heartbeats, with both proxies' quarantines, +// startReporter streams the mirror's state, with both proxies' quarantines, // to the SPECTRE server. Reporting never stops the mirror. func (s *Supervisor) startReporter(ctx context.Context) { reporter := reporter.New(s.config.Server, client.New(s.config.Server.Client), diff --git a/proto/spectre/v1/spectre.proto b/proto/spectre/v1/spectre.proto index 50da4f2..8b0b2a8 100644 --- a/proto/spectre/v1/spectre.proto +++ b/proto/spectre/v1/spectre.proto @@ -10,9 +10,9 @@ option go_package = "github.com/block/spectre/spectrepb;spectrepb"; // SpectreService collects the state of running mirrors and answers questions // about them for people and agents. service SpectreService { - // Update replaces the state of a service environment's mirror. A mirror - // sends one periodically and whenever its candidate is quarantined. - rpc Update(UpdateRequest) returns (UpdateResponse); + // Update streams a mirror's state for as long as the mirror runs. Each + // stream begins with the whole state, then changes it or keeps it alive. + rpc Update(stream UpdateRequest) returns (UpdateResponse); // ListDivergences returns the state of each mirror the server still // considers current, and the latest divergence each mirror reported. rpc ListDivergences(ListDivergencesRequest) returns (ListDivergencesResponse); @@ -31,9 +31,19 @@ enum Direction { } message UpdateRequest { - State state = 1; + oneof update { + // State replaces the mirror's whole state. Every stream begins with it. + State state = 1; + // Quarantine adds the quarantine of its direction to the state. + Quarantine quarantine = 2; + // Keepalive shows the mirror still runs when nothing has changed. + Keepalive keepalive = 3; + } } +// Keepalive carries nothing; it only keeps a quiet stream alive. +message Keepalive {} + message UpdateResponse {} // ListDivergencesRequest filters mirror states and divergences. An unset filter diff --git a/spectrepb/spectre.pb.go b/spectrepb/spectre.pb.go index bdbe307..3a52e07 100644 --- a/spectrepb/spectre.pb.go +++ b/spectrepb/spectre.pb.go @@ -140,12 +140,17 @@ func (x Quarantine_Kind) Number() protoreflect.EnumNumber { // Deprecated: Use Quarantine_Kind.Descriptor instead. func (Quarantine_Kind) EnumDescriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{7, 0} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{8, 0} } type UpdateRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - State *State `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Update: + // + // *UpdateRequest_State + // *UpdateRequest_Quarantine + // *UpdateRequest_Keepalive + Update isUpdateRequest_Update `protobuf_oneof:"update"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -180,13 +185,102 @@ func (*UpdateRequest) Descriptor() ([]byte, []int) { return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{0} } +func (x *UpdateRequest) GetUpdate() isUpdateRequest_Update { + if x != nil { + return x.Update + } + return nil +} + func (x *UpdateRequest) GetState() *State { if x != nil { - return x.State + if x, ok := x.Update.(*UpdateRequest_State); ok { + return x.State + } + } + return nil +} + +func (x *UpdateRequest) GetQuarantine() *Quarantine { + if x != nil { + if x, ok := x.Update.(*UpdateRequest_Quarantine); ok { + return x.Quarantine + } + } + return nil +} + +func (x *UpdateRequest) GetKeepalive() *Keepalive { + if x != nil { + if x, ok := x.Update.(*UpdateRequest_Keepalive); ok { + return x.Keepalive + } } return nil } +type isUpdateRequest_Update interface { + isUpdateRequest_Update() +} + +type UpdateRequest_State struct { + // State replaces the mirror's whole state. Every stream begins with it. + State *State `protobuf:"bytes,1,opt,name=state,proto3,oneof"` +} + +type UpdateRequest_Quarantine struct { + // Quarantine adds the quarantine of its direction to the state. + Quarantine *Quarantine `protobuf:"bytes,2,opt,name=quarantine,proto3,oneof"` +} + +type UpdateRequest_Keepalive struct { + // Keepalive shows the mirror still runs when nothing has changed. + Keepalive *Keepalive `protobuf:"bytes,3,opt,name=keepalive,proto3,oneof"` +} + +func (*UpdateRequest_State) isUpdateRequest_Update() {} + +func (*UpdateRequest_Quarantine) isUpdateRequest_Update() {} + +func (*UpdateRequest_Keepalive) isUpdateRequest_Update() {} + +// Keepalive carries nothing; it only keeps a quiet stream alive. +type Keepalive struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Keepalive) Reset() { + *x = Keepalive{} + mi := &file_spectre_v1_spectre_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Keepalive) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Keepalive) ProtoMessage() {} + +func (x *Keepalive) ProtoReflect() protoreflect.Message { + mi := &file_spectre_v1_spectre_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Keepalive.ProtoReflect.Descriptor instead. +func (*Keepalive) Descriptor() ([]byte, []int) { + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{1} +} + type UpdateResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -195,7 +289,7 @@ type UpdateResponse struct { func (x *UpdateResponse) Reset() { *x = UpdateResponse{} - mi := &file_spectre_v1_spectre_proto_msgTypes[1] + mi := &file_spectre_v1_spectre_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -207,7 +301,7 @@ func (x *UpdateResponse) String() string { func (*UpdateResponse) ProtoMessage() {} func (x *UpdateResponse) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[1] + mi := &file_spectre_v1_spectre_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -220,7 +314,7 @@ func (x *UpdateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateResponse.ProtoReflect.Descriptor instead. func (*UpdateResponse) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{1} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{2} } // ListDivergencesRequest filters mirror states and divergences. An unset filter @@ -238,7 +332,7 @@ type ListDivergencesRequest struct { func (x *ListDivergencesRequest) Reset() { *x = ListDivergencesRequest{} - mi := &file_spectre_v1_spectre_proto_msgTypes[2] + mi := &file_spectre_v1_spectre_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -250,7 +344,7 @@ func (x *ListDivergencesRequest) String() string { func (*ListDivergencesRequest) ProtoMessage() {} func (x *ListDivergencesRequest) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[2] + mi := &file_spectre_v1_spectre_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -263,7 +357,7 @@ func (x *ListDivergencesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListDivergencesRequest.ProtoReflect.Descriptor instead. func (*ListDivergencesRequest) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{2} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{3} } func (x *ListDivergencesRequest) GetService() string { @@ -304,7 +398,7 @@ type ListDivergencesResponse struct { func (x *ListDivergencesResponse) Reset() { *x = ListDivergencesResponse{} - mi := &file_spectre_v1_spectre_proto_msgTypes[3] + mi := &file_spectre_v1_spectre_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -316,7 +410,7 @@ func (x *ListDivergencesResponse) String() string { func (*ListDivergencesResponse) ProtoMessage() {} func (x *ListDivergencesResponse) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[3] + mi := &file_spectre_v1_spectre_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -329,7 +423,7 @@ func (x *ListDivergencesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListDivergencesResponse.ProtoReflect.Descriptor instead. func (*ListDivergencesResponse) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{3} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{4} } func (x *ListDivergencesResponse) GetStates() []*State { @@ -357,7 +451,7 @@ type ClearDivergencesRequest struct { func (x *ClearDivergencesRequest) Reset() { *x = ClearDivergencesRequest{} - mi := &file_spectre_v1_spectre_proto_msgTypes[4] + mi := &file_spectre_v1_spectre_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -369,7 +463,7 @@ func (x *ClearDivergencesRequest) String() string { func (*ClearDivergencesRequest) ProtoMessage() {} func (x *ClearDivergencesRequest) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[4] + mi := &file_spectre_v1_spectre_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -382,7 +476,7 @@ func (x *ClearDivergencesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ClearDivergencesRequest.ProtoReflect.Descriptor instead. func (*ClearDivergencesRequest) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{4} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{5} } func (x *ClearDivergencesRequest) GetService() string { @@ -407,7 +501,7 @@ type ClearDivergencesResponse struct { func (x *ClearDivergencesResponse) Reset() { *x = ClearDivergencesResponse{} - mi := &file_spectre_v1_spectre_proto_msgTypes[5] + mi := &file_spectre_v1_spectre_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -419,7 +513,7 @@ func (x *ClearDivergencesResponse) String() string { func (*ClearDivergencesResponse) ProtoMessage() {} func (x *ClearDivergencesResponse) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[5] + mi := &file_spectre_v1_spectre_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -432,7 +526,7 @@ func (x *ClearDivergencesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ClearDivergencesResponse.ProtoReflect.Descriptor instead. func (*ClearDivergencesResponse) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{5} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{6} } // Divergence is the latest quarantine of one direction of a service @@ -450,7 +544,7 @@ type Divergence struct { func (x *Divergence) Reset() { *x = Divergence{} - mi := &file_spectre_v1_spectre_proto_msgTypes[6] + mi := &file_spectre_v1_spectre_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -462,7 +556,7 @@ func (x *Divergence) String() string { func (*Divergence) ProtoMessage() {} func (x *Divergence) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[6] + mi := &file_spectre_v1_spectre_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -475,7 +569,7 @@ func (x *Divergence) ProtoReflect() protoreflect.Message { // Deprecated: Use Divergence.ProtoReflect.Descriptor instead. func (*Divergence) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{6} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{7} } func (x *Divergence) GetService() string { @@ -526,7 +620,7 @@ type Quarantine struct { func (x *Quarantine) Reset() { *x = Quarantine{} - mi := &file_spectre_v1_spectre_proto_msgTypes[7] + mi := &file_spectre_v1_spectre_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -538,7 +632,7 @@ func (x *Quarantine) String() string { func (*Quarantine) ProtoMessage() {} func (x *Quarantine) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[7] + mi := &file_spectre_v1_spectre_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -551,7 +645,7 @@ func (x *Quarantine) ProtoReflect() protoreflect.Message { // Deprecated: Use Quarantine.ProtoReflect.Descriptor instead. func (*Quarantine) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{7} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{8} } func (x *Quarantine) GetSince() *timestamppb.Timestamp { @@ -616,7 +710,7 @@ type Traffic struct { func (x *Traffic) Reset() { *x = Traffic{} - mi := &file_spectre_v1_spectre_proto_msgTypes[8] + mi := &file_spectre_v1_spectre_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -628,7 +722,7 @@ func (x *Traffic) String() string { func (*Traffic) ProtoMessage() {} func (x *Traffic) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[8] + mi := &file_spectre_v1_spectre_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -641,7 +735,7 @@ func (x *Traffic) ProtoReflect() protoreflect.Message { // Deprecated: Use Traffic.ProtoReflect.Descriptor instead. func (*Traffic) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{8} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{9} } func (x *Traffic) GetDirection() Direction { @@ -685,7 +779,7 @@ type State struct { func (x *State) Reset() { *x = State{} - mi := &file_spectre_v1_spectre_proto_msgTypes[9] + mi := &file_spectre_v1_spectre_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -697,7 +791,7 @@ func (x *State) String() string { func (*State) ProtoMessage() {} func (x *State) ProtoReflect() protoreflect.Message { - mi := &file_spectre_v1_spectre_proto_msgTypes[9] + mi := &file_spectre_v1_spectre_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -710,7 +804,7 @@ func (x *State) ProtoReflect() protoreflect.Message { // Deprecated: Use State.ProtoReflect.Descriptor instead. func (*State) Descriptor() ([]byte, []int) { - return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{9} + return file_spectre_v1_spectre_proto_rawDescGZIP(), []int{10} } func (x *State) GetService() string { @@ -760,9 +854,15 @@ var File_spectre_v1_spectre_proto protoreflect.FileDescriptor const file_spectre_v1_spectre_proto_rawDesc = "" + "\n" + "\x18spectre/v1/spectre.proto\x12\n" + - "spectre.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"8\n" + - "\rUpdateRequest\x12'\n" + - "\x05state\x18\x01 \x01(\v2\x11.spectre.v1.StateR\x05state\"\x10\n" + + "spectre.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xb5\x01\n" + + "\rUpdateRequest\x12)\n" + + "\x05state\x18\x01 \x01(\v2\x11.spectre.v1.StateH\x00R\x05state\x128\n" + + "\n" + + "quarantine\x18\x02 \x01(\v2\x16.spectre.v1.QuarantineH\x00R\n" + + "quarantine\x125\n" + + "\tkeepalive\x18\x03 \x01(\v2\x15.spectre.v1.KeepaliveH\x00R\tkeepaliveB\b\n" + + "\x06update\"\v\n" + + "\tKeepalive\"\x10\n" + "\x0eUpdateResponse\"\xca\x01\n" + "\x16ListDivergencesRequest\x12\x1d\n" + "\aservice\x18\x01 \x01(\tH\x00R\aservice\x88\x01\x01\x12%\n" + @@ -821,9 +921,9 @@ const file_spectre_v1_spectre_proto_rawDesc = "" + "\tDirection\x12\x19\n" + "\x15DIRECTION_UNSPECIFIED\x10\x00\x12\x15\n" + "\x11DIRECTION_INGRESS\x10\x01\x12\x14\n" + - "\x10DIRECTION_EGRESS\x10\x022\x8c\x02\n" + - "\x0eSpectreService\x12?\n" + - "\x06Update\x12\x19.spectre.v1.UpdateRequest\x1a\x1a.spectre.v1.UpdateResponse\x12Z\n" + + "\x10DIRECTION_EGRESS\x10\x022\x8e\x02\n" + + "\x0eSpectreService\x12A\n" + + "\x06Update\x12\x19.spectre.v1.UpdateRequest\x1a\x1a.spectre.v1.UpdateResponse(\x01\x12Z\n" + "\x0fListDivergences\x12\".spectre.v1.ListDivergencesRequest\x1a#.spectre.v1.ListDivergencesResponse\x12]\n" + "\x10ClearDivergences\x12#.spectre.v1.ClearDivergencesRequest\x1a$.spectre.v1.ClearDivergencesResponseB.Z,github.com/block/spectre/spectrepb;spectrepbb\x06proto3" @@ -840,46 +940,49 @@ func file_spectre_v1_spectre_proto_rawDescGZIP() []byte { } var file_spectre_v1_spectre_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_spectre_v1_spectre_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_spectre_v1_spectre_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_spectre_v1_spectre_proto_goTypes = []any{ (Direction)(0), // 0: spectre.v1.Direction (Quarantine_Kind)(0), // 1: spectre.v1.Quarantine.Kind (*UpdateRequest)(nil), // 2: spectre.v1.UpdateRequest - (*UpdateResponse)(nil), // 3: spectre.v1.UpdateResponse - (*ListDivergencesRequest)(nil), // 4: spectre.v1.ListDivergencesRequest - (*ListDivergencesResponse)(nil), // 5: spectre.v1.ListDivergencesResponse - (*ClearDivergencesRequest)(nil), // 6: spectre.v1.ClearDivergencesRequest - (*ClearDivergencesResponse)(nil), // 7: spectre.v1.ClearDivergencesResponse - (*Divergence)(nil), // 8: spectre.v1.Divergence - (*Quarantine)(nil), // 9: spectre.v1.Quarantine - (*Traffic)(nil), // 10: spectre.v1.Traffic - (*State)(nil), // 11: spectre.v1.State - (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 13: google.protobuf.Duration + (*Keepalive)(nil), // 3: spectre.v1.Keepalive + (*UpdateResponse)(nil), // 4: spectre.v1.UpdateResponse + (*ListDivergencesRequest)(nil), // 5: spectre.v1.ListDivergencesRequest + (*ListDivergencesResponse)(nil), // 6: spectre.v1.ListDivergencesResponse + (*ClearDivergencesRequest)(nil), // 7: spectre.v1.ClearDivergencesRequest + (*ClearDivergencesResponse)(nil), // 8: spectre.v1.ClearDivergencesResponse + (*Divergence)(nil), // 9: spectre.v1.Divergence + (*Quarantine)(nil), // 10: spectre.v1.Quarantine + (*Traffic)(nil), // 11: spectre.v1.Traffic + (*State)(nil), // 12: spectre.v1.State + (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 14: google.protobuf.Duration } var file_spectre_v1_spectre_proto_depIdxs = []int32{ - 11, // 0: spectre.v1.UpdateRequest.state:type_name -> spectre.v1.State - 11, // 1: spectre.v1.ListDivergencesResponse.states:type_name -> spectre.v1.State - 8, // 2: spectre.v1.ListDivergencesResponse.divergences:type_name -> spectre.v1.Divergence - 9, // 3: spectre.v1.Divergence.quarantine:type_name -> spectre.v1.Quarantine - 12, // 4: spectre.v1.Quarantine.since:type_name -> google.protobuf.Timestamp - 1, // 5: spectre.v1.Quarantine.kind:type_name -> spectre.v1.Quarantine.Kind - 10, // 6: spectre.v1.Quarantine.traffic:type_name -> spectre.v1.Traffic - 0, // 7: spectre.v1.Quarantine.direction:type_name -> spectre.v1.Direction - 0, // 8: spectre.v1.Traffic.direction:type_name -> spectre.v1.Direction - 13, // 9: spectre.v1.State.uptime:type_name -> google.protobuf.Duration - 9, // 10: spectre.v1.State.quarantines:type_name -> spectre.v1.Quarantine - 2, // 11: spectre.v1.SpectreService.Update:input_type -> spectre.v1.UpdateRequest - 4, // 12: spectre.v1.SpectreService.ListDivergences:input_type -> spectre.v1.ListDivergencesRequest - 6, // 13: spectre.v1.SpectreService.ClearDivergences:input_type -> spectre.v1.ClearDivergencesRequest - 3, // 14: spectre.v1.SpectreService.Update:output_type -> spectre.v1.UpdateResponse - 5, // 15: spectre.v1.SpectreService.ListDivergences:output_type -> spectre.v1.ListDivergencesResponse - 7, // 16: spectre.v1.SpectreService.ClearDivergences:output_type -> spectre.v1.ClearDivergencesResponse - 14, // [14:17] is the sub-list for method output_type - 11, // [11:14] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 12, // 0: spectre.v1.UpdateRequest.state:type_name -> spectre.v1.State + 10, // 1: spectre.v1.UpdateRequest.quarantine:type_name -> spectre.v1.Quarantine + 3, // 2: spectre.v1.UpdateRequest.keepalive:type_name -> spectre.v1.Keepalive + 12, // 3: spectre.v1.ListDivergencesResponse.states:type_name -> spectre.v1.State + 9, // 4: spectre.v1.ListDivergencesResponse.divergences:type_name -> spectre.v1.Divergence + 10, // 5: spectre.v1.Divergence.quarantine:type_name -> spectre.v1.Quarantine + 13, // 6: spectre.v1.Quarantine.since:type_name -> google.protobuf.Timestamp + 1, // 7: spectre.v1.Quarantine.kind:type_name -> spectre.v1.Quarantine.Kind + 11, // 8: spectre.v1.Quarantine.traffic:type_name -> spectre.v1.Traffic + 0, // 9: spectre.v1.Quarantine.direction:type_name -> spectre.v1.Direction + 0, // 10: spectre.v1.Traffic.direction:type_name -> spectre.v1.Direction + 14, // 11: spectre.v1.State.uptime:type_name -> google.protobuf.Duration + 10, // 12: spectre.v1.State.quarantines:type_name -> spectre.v1.Quarantine + 2, // 13: spectre.v1.SpectreService.Update:input_type -> spectre.v1.UpdateRequest + 5, // 14: spectre.v1.SpectreService.ListDivergences:input_type -> spectre.v1.ListDivergencesRequest + 7, // 15: spectre.v1.SpectreService.ClearDivergences:input_type -> spectre.v1.ClearDivergencesRequest + 4, // 16: spectre.v1.SpectreService.Update:output_type -> spectre.v1.UpdateResponse + 6, // 17: spectre.v1.SpectreService.ListDivergences:output_type -> spectre.v1.ListDivergencesResponse + 8, // 18: spectre.v1.SpectreService.ClearDivergences:output_type -> spectre.v1.ClearDivergencesResponse + 16, // [16:19] is the sub-list for method output_type + 13, // [13:16] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name } func init() { file_spectre_v1_spectre_proto_init() } @@ -887,15 +990,20 @@ func file_spectre_v1_spectre_proto_init() { if File_spectre_v1_spectre_proto != nil { return } - file_spectre_v1_spectre_proto_msgTypes[2].OneofWrappers = []any{} - file_spectre_v1_spectre_proto_msgTypes[7].OneofWrappers = []any{} + file_spectre_v1_spectre_proto_msgTypes[0].OneofWrappers = []any{ + (*UpdateRequest_State)(nil), + (*UpdateRequest_Quarantine)(nil), + (*UpdateRequest_Keepalive)(nil), + } + file_spectre_v1_spectre_proto_msgTypes[3].OneofWrappers = []any{} + file_spectre_v1_spectre_proto_msgTypes[8].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_spectre_v1_spectre_proto_rawDesc), len(file_spectre_v1_spectre_proto_rawDesc)), NumEnums: 2, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, diff --git a/spectrepb/spectrepbconnect/spectre.connect.go b/spectrepb/spectrepbconnect/spectre.connect.go index 9809ac5..2840ed3 100644 --- a/spectrepb/spectrepbconnect/spectre.connect.go +++ b/spectrepb/spectrepbconnect/spectre.connect.go @@ -37,7 +37,7 @@ const ( var ( spectreServiceUpdateSpec = sync.OnceValue(func() connect.Spec { return connect.Spec{ - StreamType: connect.StreamTypeUnary, + StreamType: connect.StreamTypeClient, Schema: spectrepb.File_spectre_v1_spectre_proto.Services().ByName("SpectreService").Methods().ByName("Update"), Procedure: SpectreServiceUpdateProcedure, } @@ -60,9 +60,9 @@ var ( // SpectreServiceClient is a client for the spectre.v1.SpectreService service. type SpectreServiceClient interface { - // Update replaces the state of a service environment's mirror. A mirror - // sends one periodically and whenever its candidate is quarantined. - Update(context.Context, *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) + // Update streams a mirror's state for as long as the mirror runs. Each + // stream begins with the whole state, then changes it or keeps it alive. + Update(context.Context) (SpectreServiceUpdateClientStream, error) // ListDivergences returns the state of each mirror the server still // considers current, and the latest divergence each mirror reported. ListDivergences(context.Context, *spectrepb.ListDivergencesRequest) (*spectrepb.ListDivergencesResponse, error) @@ -77,11 +77,40 @@ func NewSpectreServiceClient(client *connect.Client) SpectreServiceClient { return &spectreServiceClient{client: client} } +// SpectreServiceUpdateClientStream is the client stream for the SpectreService's Update RPC. +type SpectreServiceUpdateClientStream struct { + stream connect.ClientStream +} + +// SendHeaders opens the stream and flushes the request headers without a message. The first Send or +// Receive does this implicitly. +func (s SpectreServiceUpdateClientStream) SendHeaders() error { + return s.stream.SendHeaders() +} + +// Send sends a request message to the server. +func (s SpectreServiceUpdateClientStream) Send(req *spectrepb.UpdateRequest) error { + return s.stream.Send(req) +} + +// CloseAndReceive closes the request side of the stream and returns the single response message. It +// reads the stream to completion to release its resources. +func (s SpectreServiceUpdateClientStream) CloseAndReceive() (*spectrepb.UpdateResponse, error) { + if err := s.stream.CloseSend(); err != nil { + return nil, err + } + var res spectrepb.UpdateResponse + if err := s.stream.Receive(&res); err != nil { + return nil, err + } + return &res, nil +} + // SpectreServiceHandler is an implementation of the spectre.v1.SpectreService service. type SpectreServiceHandler interface { - // Update replaces the state of a service environment's mirror. A mirror - // sends one periodically and whenever its candidate is quarantined. - Update(context.Context, *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) + // Update streams a mirror's state for as long as the mirror runs. Each + // stream begins with the whole state, then changes it or keeps it alive. + Update(context.Context, SpectreServiceUpdateServerStream) (*spectrepb.UpdateResponse, error) // ListDivergences returns the state of each mirror the server still // considers current, and the latest divergence each mirror reported. ListDivergences(context.Context, *spectrepb.ListDivergencesRequest) (*spectrepb.ListDivergencesResponse, error) @@ -101,10 +130,24 @@ func RegisterSpectreServiceHandler(server *connect.Server, svc SpectreServiceHan ) } +// SpectreServiceUpdateServerStream is the server stream for the SpectreService's Update RPC. +type SpectreServiceUpdateServerStream struct { + stream connect.ServerStream +} + +// Receive returns the next request message from the client. +func (s SpectreServiceUpdateServerStream) Receive() (*spectrepb.UpdateRequest, error) { + var req spectrepb.UpdateRequest + if err := s.stream.Receive(&req); err != nil { + return nil, err + } + return &req, nil +} + // UnimplementedSpectreServiceHandler returns CodeUnimplemented from all methods. type UnimplementedSpectreServiceHandler struct{} -func (UnimplementedSpectreServiceHandler) Update(context.Context, *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) { +func (UnimplementedSpectreServiceHandler) Update(context.Context, SpectreServiceUpdateServerStream) (*spectrepb.UpdateResponse, error) { return nil, connect.NewError(connect.CodeUnimplemented, "spectre.v1.SpectreService.Update is not implemented") } @@ -120,12 +163,12 @@ type spectreServiceClient struct { client *connect.Client } -func (c *spectreServiceClient) Update(ctx context.Context, req *spectrepb.UpdateRequest) (*spectrepb.UpdateResponse, error) { - var res spectrepb.UpdateResponse - if err := c.client.CallUnary(ctx, spectreServiceUpdateSpec(), req, &res); err != nil { - return nil, err +func (c *spectreServiceClient) Update(ctx context.Context) (SpectreServiceUpdateClientStream, error) { + stream, err := c.client.CallClientStream(ctx, spectreServiceUpdateSpec()) + if err != nil { + return SpectreServiceUpdateClientStream{}, err } - return &res, nil + return SpectreServiceUpdateClientStream{stream: stream}, nil } func (c *spectreServiceClient) ListDivergences(ctx context.Context, req *spectrepb.ListDivergencesRequest) (*spectrepb.ListDivergencesResponse, error) { @@ -147,11 +190,7 @@ func (c *spectreServiceClient) ClearDivergences(ctx context.Context, req *spectr type spectreServiceHandler struct{ svc SpectreServiceHandler } func (h spectreServiceHandler) update(ctx context.Context, _ connect.Spec, stream connect.ServerStream) error { - var req spectrepb.UpdateRequest - if err := stream.Receive(&req); err != nil { - return err - } - res, err := h.svc.Update(ctx, &req) + res, err := h.svc.Update(ctx, SpectreServiceUpdateServerStream{stream: stream}) if err != nil { return err }