From 285ca203f2e13b8c5e13c59b0273ee2c7819765b Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 5 Nov 2026 20:08:11 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20fix:=20stop=20a=20rejected?= =?UTF-8?q?=20stream=20from=20drawing=20shared=20calls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream a server rejected at once could stay selectable in two ways. Its receiver could end it before open added it to the group, leaving it there dead until it expired. Or Send could report its end before the receiver ran, so records handed back chose it again until their attempts ran out. Either way calls fell back to streams of their own and kept reaching the draining server. Receiving now starts once the group holds the stream, and a stream leaves its group as soon as Send reports its end. --- lanes.go | 18 +++++-- lanes_test.go | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/lanes.go b/lanes.go index 6f2a4a7..cec907b 100644 --- a/lanes.go +++ b/lanes.go @@ -412,7 +412,6 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) } opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), outstanding: make(map[uint64]*laneCall)} opened.load.Store(1) - go opened.receive() if key.owner != 0 { group.writers.Add(1) } @@ -450,6 +449,12 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) close(group.opening) group.opening = nil p.mu.Unlock() + // A stream that ends leaves its group, so it receives only once the group + // holds it. One a server rejected at once would otherwise stay there dead, + // looking idle, and send every call to a stream of its own until it expired. + if opened != nil { + go opened.receive() + } for _, stream := range retired { stream.drain() } @@ -610,10 +615,15 @@ func (l *lane) writeNext() bool { return true } -// sendFailed ends the stream after a local send failure, which may never -// surface through Recv. io.EOF means Recv holds the stream's final status. +// sendFailed stops using a stream whose send failed. io.EOF means the stream has +// ended and Recv holds its final status, so it leaves its group now: the records +// handed back must not choose it again while the receiver has yet to run. Any other +// error is a local failure, which may never surface through Recv. func (l *lane) sendFailed(err error) { - if err != nil && !errors.Is(err, io.EOF) { + switch { + case errors.Is(err, io.EOF): + l.pool.remove(l) + case err != nil: l.cancel() } } diff --git a/lanes_test.go b/lanes_test.go index 0324d15..a6d9320 100644 --- a/lanes_test.go +++ b/lanes_test.go @@ -572,6 +572,134 @@ func TestLoneStreamMovesOffADrainingServerAtOnce(t *testing.T) { // A Client that knows its servers probes one that has not refused; behind one // address, a probe can still reach the draining server. +// scriptedStreamsClient opens the streams it holds, one per Execute. +type scriptedStreamsClient struct { + pb.StoreServiceClient + mu sync.Mutex + streams []grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] + opened int +} + +func (c *scriptedStreamsClient) Execute(context.Context, ...grpc.CallOption) (grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses], error) { + c.mu.Lock() + defer c.mu.Unlock() + stream := c.streams[c.opened] + c.opened++ + return stream, nil +} + +// rejectedStream is a stream a server has already rejected: Send reports that it +// ended, and Recv returns the rejection once hold closes. +type rejectedStream struct { + grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] + ctx context.Context + // receiving closes when the client first asks for the stream's result. + receiving chan struct{} + hold chan struct{} +} + +func (s *rejectedStream) Context() context.Context { return s.ctx } + +func (s *rejectedStream) Send(*pb.ExecuteRequests) error { return io.EOF } + +func (s *rejectedStream) Recv() (*pb.ExecuteResponses, error) { + close(s.receiving) + <-s.hold + return nil, status.Error(codes.Unavailable, "draining") +} + +// applyingStream answers every record as applied. +type applyingStream struct { + grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] + ctx context.Context + replies chan *pb.ExecuteResponses +} + +func (s *applyingStream) Context() context.Context { return s.ctx } + +func (s *applyingStream) Send(message *pb.ExecuteRequests) error { + for _, request := range message.Requests { + result := protocol.Mutation(pb.MutationOutcome_APPLIED, nil) + value := &pb.Event_MutationResult{MutationResult: result} + event := &pb.Event{Value: value} + response := &pb.ExecuteResponse{Index: request.Index, Event: event} + reply := &pb.ExecuteResponses{Responses: []*pb.ExecuteResponse{response}} + s.replies <- reply + } + return nil +} + +func (s *applyingStream) Recv() (*pb.ExecuteResponses, error) { + select { + case reply := <-s.replies: + return reply, nil + case <-s.ctx.Done(): + return nil, s.ctx.Err() + } +} + +// A stream whose Send reported its end leaves its group at once. Until its receiver +// runs nothing else says it ended, and a record sent back to it would run out of +// attempts and use a stream of its own, which may reach a draining server. +func TestStreamWhoseSendEndedTakesNoMoreRecords(t *testing.T) { + rejected := &rejectedStream{ctx: t.Context(), receiving: make(chan struct{}), hold: make(chan struct{})} + t.Cleanup(func() { close(rejected.hold) }) + applying := &applyingStream{ctx: t.Context(), replies: make(chan *pb.ExecuteResponses, 1)} + client := &scriptedStreamsClient{streams: []grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses]{rejected, applying}} + pool := newLanePool(client, nil) + t.Cleanup(pool.close) + command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil) + if err != nil { + t.Fatal(err) + } + key := laneKey{store: "records"} + event, shared, err := pool.finish(t.Context(), key, command, nil) + if err != nil || !shared || event.GetMutationResult().GetOutcome() != MutationApplied { + t.Fatal("record did not move to the next shared stream", event, shared, err) + } + if client.opened != 2 { + t.Fatal("streams opened", client.opened) + } +} + +// A stream that ends leaves its group. One a server rejects at once must not end +// before the group holds it, or it stays there dead until it expires. +func TestRejectedStreamDoesNotStayInItsGroup(t *testing.T) { + hold := make(chan struct{}) + close(hold) + stream := &rejectedStream{ctx: t.Context(), receiving: make(chan struct{}), hold: hold} + client := &scriptedStreamsClient{streams: []grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses]{stream}} + pool := newLanePool(client, nil) + t.Cleanup(pool.close) + key := laneKey{store: "records"} + group := &laneGroup{opening: make(chan struct{})} + pool.groups[key] = group + opening := laneOpening{key: key, group: group} + opened := make(chan *lane, 1) + // Holding the pool's lock stops open before it can add the stream. + pool.mu.Lock() + go func() { + stream, err := pool.open(t.Context(), opening) + if err != nil { + t.Error(err) + } + opened <- stream + }() + select { + case <-stream.receiving: + t.Error("stream received before its group held it") + case <-time.After(50 * time.Millisecond): + } + pool.mu.Unlock() + // A stream's context ends after it has left its group. + <-(<-opened).ctx.Done() + pool.mu.Lock() + defer pool.mu.Unlock() + if len(group.lanes) != 0 { + t.Fatal("a rejected stream stayed in its group", len(group.lanes)) + } +} + func TestRejectedProbeDoesNotHoldCallsOnADrainingServer(t *testing.T) { first, second := newLaneTestPeer(), newLaneTestPeer() client := laneTestInTurn(t, first, second) From f921856ecddd0e238a291905eca900a27e46be30 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 5 Nov 2026 20:46:10 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20fix:=20hand=20records=20back?= =?UTF-8?q?=20only=20after=20an=20ended=20stream's=20receiver=20ran?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the stream as soon as Send reported its end left its refusal unrecorded, so a retry could ask for the same draining server, now the one with the fewest streams. Waiting for the receiver settles both at once. --- lanes.go | 14 +++-- lanes_test.go | 165 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 126 insertions(+), 53 deletions(-) diff --git a/lanes.go b/lanes.go index cec907b..0135d14 100644 --- a/lanes.go +++ b/lanes.go @@ -191,6 +191,8 @@ type lane struct { answered bool // wake asks the writer to send what is queued. wake chan struct{} + // received closes once the receiver has ended the stream. + received chan struct{} mu sync.Mutex queued []*laneCall @@ -410,7 +412,7 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) if chosen := picked.Load(); chosen != nil { address = *chosen } - opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), outstanding: make(map[uint64]*laneCall)} + opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), received: make(chan struct{}), outstanding: make(map[uint64]*laneCall)} opened.load.Store(1) if key.owner != 0 { group.writers.Add(1) @@ -616,13 +618,14 @@ func (l *lane) writeNext() bool { } // sendFailed stops using a stream whose send failed. io.EOF means the stream has -// ended and Recv holds its final status, so it leaves its group now: the records -// handed back must not choose it again while the receiver has yet to run. Any other -// error is a local failure, which may never surface through Recv. +// ended and Recv holds its final status, so it waits for the receiver to read it: +// only then has the stream left its group and any refusal counted, which the records +// handed back need in order to choose another stream and server. Any other error is +// a local failure, which may never surface through Recv. func (l *lane) sendFailed(err error) { switch { case errors.Is(err, io.EOF): - l.pool.remove(l) + <-l.received case err != nil: l.cancel() } @@ -691,6 +694,7 @@ func (l *lane) receive() { l.pool.refusedBy(l.address) } l.fail(err) + close(l.received) } // ended is the outcome of a record outstanding when the stream ended. A record diff --git a/lanes_test.go b/lanes_test.go index a6d9320..e1cf04d 100644 --- a/lanes_test.go +++ b/lanes_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "slices" "strings" "sync" "sync/atomic" @@ -570,54 +571,61 @@ func TestLoneStreamMovesOffADrainingServerAtOnce(t *testing.T) { } } -// A Client that knows its servers probes one that has not refused; behind one -// address, a probe can still reach the draining server. -// scriptedStreamsClient opens the streams it holds, one per Execute. +// scriptedStreamsClient opens the streams it holds, one per Execute. Like the +// picker, it reports the server a stream asked for as the one it chose. type scriptedStreamsClient struct { pb.StoreServiceClient mu sync.Mutex - streams []grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] - opened int + streams []*scriptedStream + asked []string } -func (c *scriptedStreamsClient) Execute(context.Context, ...grpc.CallOption) (grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses], error) { +func (c *scriptedStreamsClient) Execute(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses], error) { c.mu.Lock() defer c.mu.Unlock() - stream := c.streams[c.opened] - c.opened++ + target, _ := ctx.Value(endpointPreference).(string) + if picked, ok := ctx.Value(endpointPicked).(*atomic.Pointer[string]); ok { + picked.Store(&target) + } + stream := c.streams[len(c.asked)] + stream.ctx = ctx + c.asked = append(c.asked, target) return stream, nil } -// rejectedStream is a stream a server has already rejected: Send reports that it -// ended, and Recv returns the rejection once hold closes. -type rejectedStream struct { - grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] - ctx context.Context - // receiving closes when the client first asks for the stream's result. - receiving chan struct{} - hold chan struct{} +func (c *scriptedStreamsClient) servers() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.asked) } -func (s *rejectedStream) Context() context.Context { return s.ctx } - -func (s *rejectedStream) Send(*pb.ExecuteRequests) error { return io.EOF } - -func (s *rejectedStream) Recv() (*pb.ExecuteResponses, error) { - close(s.receiving) - <-s.hold - return nil, status.Error(codes.Unavailable, "draining") +// scriptedStream answers every record as applied. With hold it is a stream a server +// has already rejected: Send reports that it ended, and Recv returns the rejection +// once hold closes. +type scriptedStream struct { + grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] + ctx context.Context + hold chan struct{} + // sending and receiving close at the first Send and Recv. + sending, receiving chan struct{} + sendOnce, recvOnce sync.Once + replies chan *pb.ExecuteResponses } -// applyingStream answers every record as applied. -type applyingStream struct { - grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses] - ctx context.Context - replies chan *pb.ExecuteResponses +func newScriptedStream(hold chan struct{}) *scriptedStream { + stream := &scriptedStream{hold: hold, sending: make(chan struct{}), receiving: make(chan struct{}), replies: make(chan *pb.ExecuteResponses, 1)} + return stream } -func (s *applyingStream) Context() context.Context { return s.ctx } +func (s *scriptedStream) Context() context.Context { return s.ctx } -func (s *applyingStream) Send(message *pb.ExecuteRequests) error { +func (s *scriptedStream) CloseSend() error { return nil } + +func (s *scriptedStream) Send(message *pb.ExecuteRequests) error { + s.sendOnce.Do(func() { close(s.sending) }) + if s.hold != nil { + return io.EOF + } for _, request := range message.Requests { result := protocol.Mutation(pb.MutationOutcome_APPLIED, nil) value := &pb.Event_MutationResult{MutationResult: result} @@ -629,8 +637,11 @@ func (s *applyingStream) Send(message *pb.ExecuteRequests) error { return nil } -func (s *applyingStream) Recv() (*pb.ExecuteResponses, error) { +func (s *scriptedStream) Recv() (*pb.ExecuteResponses, error) { + s.recvOnce.Do(func() { close(s.receiving) }) select { + case <-s.hold: + return nil, status.Error(codes.Unavailable, "draining") case reply := <-s.replies: return reply, nil case <-s.ctx.Done(): @@ -638,27 +649,83 @@ func (s *applyingStream) Recv() (*pb.ExecuteResponses, error) { } } -// A stream whose Send reported its end leaves its group at once. Until its receiver -// runs nothing else says it ended, and a record sent back to it would run out of -// attempts and use a stream of its own, which may reach a draining server. -func TestStreamWhoseSendEndedTakesNoMoreRecords(t *testing.T) { - rejected := &rejectedStream{ctx: t.Context(), receiving: make(chan struct{}), hold: make(chan struct{})} - t.Cleanup(func() { close(rejected.hold) }) - applying := &applyingStream{ctx: t.Context(), replies: make(chan *pb.ExecuteResponses, 1)} - client := &scriptedStreamsClient{streams: []grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses]{rejected, applying}} - pool := newLanePool(client, nil) +// A stream whose Send reported its end still holds the status that says whether its +// server refused. A record it hands back waits for that. Sooner, the record would go +// back to the same stream until it ran out of attempts, or ask for the same server +// again as the one with the fewest streams. +func TestEndedStreamHandsItsRecordBackOnceItsRefusalCounts(t *testing.T) { + const draining, healthy = "10.0.0.1:1", "10.0.0.2:1" + servers := &endpoints{resolved: []string{draining, healthy}, ready: []string{draining, healthy}} + hold := make(chan struct{}) + rejected := newScriptedStream(hold) + client := &scriptedStreamsClient{streams: []*scriptedStream{rejected, newScriptedStream(nil)}} + pool := newLanePool(client, servers) t.Cleanup(pool.close) command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil) if err != nil { t.Fatal(err) } + // A busy stream on the healthy server makes the group want another stream, on + // the server with the fewest. key := laneKey{store: "records"} - event, shared, err := pool.finish(t.Context(), key, command, nil) - if err != nil || !shared || event.GetMutationResult().GetOutcome() != MutationApplied { - t.Fatal("record did not move to the next shared stream", event, shared, err) + busy := &lane{address: healthy, expires: time.Now().Add(laneLifetime)} + busy.load.Store(int64(laneOpenAt)) + group := &laneGroup{lanes: []*lane{busy}} + pool.groups[key] = group + type finished struct { + event *pb.Event + shared bool + err error } - if client.opened != 2 { - t.Fatal("streams opened", client.opened) + done := make(chan finished, 1) + go func() { + event, shared, err := pool.finish(t.Context(), key, command, nil) + result := finished{event: event, shared: shared, err: err} + done <- result + }() + <-rejected.sending + select { + case result := <-done: + t.Fatal("record left a stream whose receiver had not run", result) + case <-time.After(50 * time.Millisecond): + } + close(hold) + result := <-done + if result.err != nil || !result.shared || result.event.GetMutationResult().GetOutcome() != MutationApplied { + t.Fatal("record did not move to a shared stream", result) + } + if asked := client.servers(); !slices.Equal(asked, []string{draining, healthy}) { + t.Fatal("record asked for the server that refused it", asked) + } +} + +// A record stream that failed joins its writers, one of which can be waiting for +// the receiver of a stream that ended. Canceling the stream ends that wait. +func TestFailedRecordStreamJoinsAWriterWaitingForItsReceiver(t *testing.T) { + rejected := newScriptedStream(make(chan struct{})) + client := &scriptedStreamsClient{streams: []*scriptedStream{rejected}} + pool := newLanePool(client, nil) + t.Cleanup(pool.close) + command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil) + if err != nil { + t.Fatal(err) + } + key := pool.ownKey("records", false) + queued := pool.enqueue(t.Context(), key, command) + if queued.lane == nil { + t.Fatal("no stream took the record", queued.err) + } + queued.lane.signal() + <-rejected.sending + forgotten := make(chan struct{}) + go func() { + pool.forget(key, true) + close(forgotten) + }() + select { + case <-forgotten: + case <-time.After(5 * time.Second): + t.Fatal("a failed record stream hung on a stream that ended") } } @@ -667,8 +734,8 @@ func TestStreamWhoseSendEndedTakesNoMoreRecords(t *testing.T) { func TestRejectedStreamDoesNotStayInItsGroup(t *testing.T) { hold := make(chan struct{}) close(hold) - stream := &rejectedStream{ctx: t.Context(), receiving: make(chan struct{}), hold: hold} - client := &scriptedStreamsClient{streams: []grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses]{stream}} + stream := newScriptedStream(hold) + client := &scriptedStreamsClient{streams: []*scriptedStream{stream}} pool := newLanePool(client, nil) t.Cleanup(pool.close) key := laneKey{store: "records"} @@ -700,6 +767,8 @@ func TestRejectedStreamDoesNotStayInItsGroup(t *testing.T) { } } +// A Client that knows its servers probes one that has not refused; behind one +// address, a probe can still reach the draining server. func TestRejectedProbeDoesNotHoldCallsOnADrainingServer(t *testing.T) { first, second := newLaneTestPeer(), newLaneTestPeer() client := laneTestInTurn(t, first, second) From 0e0d57176568aaa821ae1c8cd5ff699d7c333d74 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 5 Nov 2026 21:07:21 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=93=9D=20docs:=20point=20the=20quick?= =?UTF-8?q?=20start=20at=20v0.24.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 375f43a..42bd18b 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ This writes one record to MongoDB through Weir and reads it back. mkdir hello cd hello go mod init example.com/hello -go get github.com/batchstream/weir-go@v0.24.0 go.mongodb.org/mongo-driver/v2/bson +go get github.com/batchstream/weir-go@v0.24.1 go.mongodb.org/mongo-driver/v2/bson ``` **Step 3.** Save this as `main.go`.