diff --git a/README.md b/README.md index 375f43a..42bd18b 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ This writes one record to MongoDB through Weir and reads it back. mkdir hello cd hello go mod init example.com/hello -go get github.com/batchstream/weir-go@v0.24.0 go.mongodb.org/mongo-driver/v2/bson +go get github.com/batchstream/weir-go@v0.24.1 go.mongodb.org/mongo-driver/v2/bson ``` **Step 3.** Save this as `main.go`. diff --git a/lanes.go b/lanes.go index 6f2a4a7..0135d14 100644 --- a/lanes.go +++ b/lanes.go @@ -191,6 +191,8 @@ type lane struct { answered bool // wake asks the writer to send what is queued. wake chan struct{} + // received closes once the receiver has ended the stream. + received chan struct{} mu sync.Mutex queued []*laneCall @@ -410,9 +412,8 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) if chosen := picked.Load(); chosen != nil { address = *chosen } - opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), outstanding: make(map[uint64]*laneCall)} + opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), received: make(chan struct{}), outstanding: make(map[uint64]*laneCall)} opened.load.Store(1) - go opened.receive() if key.owner != 0 { group.writers.Add(1) } @@ -450,6 +451,12 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) close(group.opening) group.opening = nil p.mu.Unlock() + // A stream that ends leaves its group, so it receives only once the group + // holds it. One a server rejected at once would otherwise stay there dead, + // looking idle, and send every call to a stream of its own until it expired. + if opened != nil { + go opened.receive() + } for _, stream := range retired { stream.drain() } @@ -610,10 +617,16 @@ func (l *lane) writeNext() bool { return true } -// sendFailed ends the stream after a local send failure, which may never -// surface through Recv. io.EOF means Recv holds the stream's final status. +// sendFailed stops using a stream whose send failed. io.EOF means the stream has +// ended and Recv holds its final status, so it waits for the receiver to read it: +// only then has the stream left its group and any refusal counted, which the records +// handed back need in order to choose another stream and server. Any other error is +// a local failure, which may never surface through Recv. func (l *lane) sendFailed(err error) { - if err != nil && !errors.Is(err, io.EOF) { + switch { + case errors.Is(err, io.EOF): + <-l.received + case err != nil: l.cancel() } } @@ -681,6 +694,7 @@ func (l *lane) receive() { l.pool.refusedBy(l.address) } l.fail(err) + close(l.received) } // ended is the outcome of a record outstanding when the stream ended. A record diff --git a/lanes_test.go b/lanes_test.go index 0324d15..e1cf04d 100644 --- a/lanes_test.go +++ b/lanes_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "slices" "strings" "sync" "sync/atomic" @@ -570,6 +571,202 @@ func TestLoneStreamMovesOffADrainingServerAtOnce(t *testing.T) { } } +// scriptedStreamsClient opens the streams it holds, one per Execute. Like the +// picker, it reports the server a stream asked for as the one it chose. +type scriptedStreamsClient struct { + pb.StoreServiceClient + mu sync.Mutex + streams []*scriptedStream + asked []string +} + +func (c *scriptedStreamsClient) Execute(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses], error) { + c.mu.Lock() + defer c.mu.Unlock() + target, _ := ctx.Value(endpointPreference).(string) + if picked, ok := ctx.Value(endpointPicked).(*atomic.Pointer[string]); ok { + picked.Store(&target) + } + stream := c.streams[len(c.asked)] + stream.ctx = ctx + c.asked = append(c.asked, target) + return stream, nil +} + +func (c *scriptedStreamsClient) servers() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.asked) +} + +// scriptedStream answers every record as applied. With hold it is a stream a server +// has already rejected: Send reports that it ended, and Recv returns the rejection +// once hold closes. +type scriptedStream struct { + grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] + ctx context.Context + hold chan struct{} + // sending and receiving close at the first Send and Recv. + sending, receiving chan struct{} + sendOnce, recvOnce sync.Once + replies chan *pb.ExecuteResponses +} + +func newScriptedStream(hold chan struct{}) *scriptedStream { + stream := &scriptedStream{hold: hold, sending: make(chan struct{}), receiving: make(chan struct{}), replies: make(chan *pb.ExecuteResponses, 1)} + return stream +} + +func (s *scriptedStream) Context() context.Context { return s.ctx } + +func (s *scriptedStream) CloseSend() error { return nil } + +func (s *scriptedStream) Send(message *pb.ExecuteRequests) error { + s.sendOnce.Do(func() { close(s.sending) }) + if s.hold != nil { + return io.EOF + } + for _, request := range message.Requests { + result := protocol.Mutation(pb.MutationOutcome_APPLIED, nil) + value := &pb.Event_MutationResult{MutationResult: result} + event := &pb.Event{Value: value} + response := &pb.ExecuteResponse{Index: request.Index, Event: event} + reply := &pb.ExecuteResponses{Responses: []*pb.ExecuteResponse{response}} + s.replies <- reply + } + return nil +} + +func (s *scriptedStream) Recv() (*pb.ExecuteResponses, error) { + s.recvOnce.Do(func() { close(s.receiving) }) + select { + case <-s.hold: + return nil, status.Error(codes.Unavailable, "draining") + case reply := <-s.replies: + return reply, nil + case <-s.ctx.Done(): + return nil, s.ctx.Err() + } +} + +// A stream whose Send reported its end still holds the status that says whether its +// server refused. A record it hands back waits for that. Sooner, the record would go +// back to the same stream until it ran out of attempts, or ask for the same server +// again as the one with the fewest streams. +func TestEndedStreamHandsItsRecordBackOnceItsRefusalCounts(t *testing.T) { + const draining, healthy = "10.0.0.1:1", "10.0.0.2:1" + servers := &endpoints{resolved: []string{draining, healthy}, ready: []string{draining, healthy}} + hold := make(chan struct{}) + rejected := newScriptedStream(hold) + client := &scriptedStreamsClient{streams: []*scriptedStream{rejected, newScriptedStream(nil)}} + pool := newLanePool(client, servers) + t.Cleanup(pool.close) + command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil) + if err != nil { + t.Fatal(err) + } + // A busy stream on the healthy server makes the group want another stream, on + // the server with the fewest. + key := laneKey{store: "records"} + busy := &lane{address: healthy, expires: time.Now().Add(laneLifetime)} + busy.load.Store(int64(laneOpenAt)) + group := &laneGroup{lanes: []*lane{busy}} + pool.groups[key] = group + type finished struct { + event *pb.Event + shared bool + err error + } + done := make(chan finished, 1) + go func() { + event, shared, err := pool.finish(t.Context(), key, command, nil) + result := finished{event: event, shared: shared, err: err} + done <- result + }() + <-rejected.sending + select { + case result := <-done: + t.Fatal("record left a stream whose receiver had not run", result) + case <-time.After(50 * time.Millisecond): + } + close(hold) + result := <-done + if result.err != nil || !result.shared || result.event.GetMutationResult().GetOutcome() != MutationApplied { + t.Fatal("record did not move to a shared stream", result) + } + if asked := client.servers(); !slices.Equal(asked, []string{draining, healthy}) { + t.Fatal("record asked for the server that refused it", asked) + } +} + +// A record stream that failed joins its writers, one of which can be waiting for +// the receiver of a stream that ended. Canceling the stream ends that wait. +func TestFailedRecordStreamJoinsAWriterWaitingForItsReceiver(t *testing.T) { + rejected := newScriptedStream(make(chan struct{})) + client := &scriptedStreamsClient{streams: []*scriptedStream{rejected}} + pool := newLanePool(client, nil) + t.Cleanup(pool.close) + command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil) + if err != nil { + t.Fatal(err) + } + key := pool.ownKey("records", false) + queued := pool.enqueue(t.Context(), key, command) + if queued.lane == nil { + t.Fatal("no stream took the record", queued.err) + } + queued.lane.signal() + <-rejected.sending + forgotten := make(chan struct{}) + go func() { + pool.forget(key, true) + close(forgotten) + }() + select { + case <-forgotten: + case <-time.After(5 * time.Second): + t.Fatal("a failed record stream hung on a stream that ended") + } +} + +// A stream that ends leaves its group. One a server rejects at once must not end +// before the group holds it, or it stays there dead until it expires. +func TestRejectedStreamDoesNotStayInItsGroup(t *testing.T) { + hold := make(chan struct{}) + close(hold) + stream := newScriptedStream(hold) + client := &scriptedStreamsClient{streams: []*scriptedStream{stream}} + pool := newLanePool(client, nil) + t.Cleanup(pool.close) + key := laneKey{store: "records"} + group := &laneGroup{opening: make(chan struct{})} + pool.groups[key] = group + opening := laneOpening{key: key, group: group} + opened := make(chan *lane, 1) + // Holding the pool's lock stops open before it can add the stream. + pool.mu.Lock() + go func() { + stream, err := pool.open(t.Context(), opening) + if err != nil { + t.Error(err) + } + opened <- stream + }() + select { + case <-stream.receiving: + t.Error("stream received before its group held it") + case <-time.After(50 * time.Millisecond): + } + pool.mu.Unlock() + // A stream's context ends after it has left its group. + <-(<-opened).ctx.Done() + pool.mu.Lock() + defer pool.mu.Unlock() + if len(group.lanes) != 0 { + t.Fatal("a rejected stream stayed in its group", len(group.lanes)) + } +} + // A Client that knows its servers probes one that has not refused; behind one // address, a probe can still reach the draining server. func TestRejectedProbeDoesNotHoldCallsOnADrainingServer(t *testing.T) {