Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ This writes one record to MongoDB through Weir and reads it back.
mkdir hello
cd hello
go mod init example.com/hello
go get github.com/batchstream/weir-go@v0.24.0 go.mongodb.org/mongo-driver/v2/bson
go get github.com/batchstream/weir-go@v0.24.1 go.mongodb.org/mongo-driver/v2/bson
```

**Step 3.** Save this as `main.go`.
Expand Down
24 changes: 19 additions & 5 deletions lanes.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,8 @@ type lane struct {
answered bool
// wake asks the writer to send what is queued.
wake chan struct{}
// received closes once the receiver has ended the stream.
received chan struct{}

mu sync.Mutex
queued []*laneCall
Expand Down Expand Up @@ -410,9 +412,8 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error)
if chosen := picked.Load(); chosen != nil {
address = *chosen
}
opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), outstanding: make(map[uint64]*laneCall)}
opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), received: make(chan struct{}), outstanding: make(map[uint64]*laneCall)}
opened.load.Store(1)
go opened.receive()
if key.owner != 0 {
group.writers.Add(1)
}
Expand Down Expand Up @@ -450,6 +451,12 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error)
close(group.opening)
group.opening = nil
p.mu.Unlock()
// A stream that ends leaves its group, so it receives only once the group
// holds it. One a server rejected at once would otherwise stay there dead,
// looking idle, and send every call to a stream of its own until it expired.
if opened != nil {
go opened.receive()
}
for _, stream := range retired {
stream.drain()
}
Expand Down Expand Up @@ -610,10 +617,16 @@ func (l *lane) writeNext() bool {
return true
}

// sendFailed ends the stream after a local send failure, which may never
// surface through Recv. io.EOF means Recv holds the stream's final status.
// sendFailed stops using a stream whose send failed. io.EOF means the stream has
// ended and Recv holds its final status, so it waits for the receiver to read it:
// only then has the stream left its group and any refusal counted, which the records
// handed back need in order to choose another stream and server. Any other error is
// a local failure, which may never surface through Recv.
func (l *lane) sendFailed(err error) {
if err != nil && !errors.Is(err, io.EOF) {
switch {
case errors.Is(err, io.EOF):
<-l.received
case err != nil:
l.cancel()
}
}
Expand Down Expand Up @@ -681,6 +694,7 @@ func (l *lane) receive() {
l.pool.refusedBy(l.address)
}
l.fail(err)
close(l.received)
}

// ended is the outcome of a record outstanding when the stream ended. A record
Expand Down
197 changes: 197 additions & 0 deletions lanes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net"
"slices"
"strings"
"sync"
"sync/atomic"
Expand Down Expand Up @@ -570,6 +571,202 @@ func TestLoneStreamMovesOffADrainingServerAtOnce(t *testing.T) {
}
}

// scriptedStreamsClient opens the streams it holds, one per Execute. Like the
// picker, it reports the server a stream asked for as the one it chose.
type scriptedStreamsClient struct {
pb.StoreServiceClient
mu sync.Mutex
streams []*scriptedStream
asked []string
}

func (c *scriptedStreamsClient) Execute(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses], error) {
c.mu.Lock()
defer c.mu.Unlock()
target, _ := ctx.Value(endpointPreference).(string)
if picked, ok := ctx.Value(endpointPicked).(*atomic.Pointer[string]); ok {
picked.Store(&target)
}
stream := c.streams[len(c.asked)]
stream.ctx = ctx
c.asked = append(c.asked, target)
return stream, nil
}

func (c *scriptedStreamsClient) servers() []string {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.asked)
}

// scriptedStream answers every record as applied. With hold it is a stream a server
// has already rejected: Send reports that it ended, and Recv returns the rejection
// once hold closes.
type scriptedStream struct {
grpc.BidiStreamingClient[pb.ExecuteRequests, pb.ExecuteResponses]
ctx context.Context
hold chan struct{}
// sending and receiving close at the first Send and Recv.
sending, receiving chan struct{}
sendOnce, recvOnce sync.Once
replies chan *pb.ExecuteResponses
}

func newScriptedStream(hold chan struct{}) *scriptedStream {
stream := &scriptedStream{hold: hold, sending: make(chan struct{}), receiving: make(chan struct{}), replies: make(chan *pb.ExecuteResponses, 1)}
return stream
}

func (s *scriptedStream) Context() context.Context { return s.ctx }

func (s *scriptedStream) CloseSend() error { return nil }

func (s *scriptedStream) Send(message *pb.ExecuteRequests) error {
s.sendOnce.Do(func() { close(s.sending) })
if s.hold != nil {
return io.EOF
}
for _, request := range message.Requests {
result := protocol.Mutation(pb.MutationOutcome_APPLIED, nil)
value := &pb.Event_MutationResult{MutationResult: result}
event := &pb.Event{Value: value}
response := &pb.ExecuteResponse{Index: request.Index, Event: event}
reply := &pb.ExecuteResponses{Responses: []*pb.ExecuteResponse{response}}
s.replies <- reply
}
return nil
}

func (s *scriptedStream) Recv() (*pb.ExecuteResponses, error) {
s.recvOnce.Do(func() { close(s.receiving) })
select {
case <-s.hold:
return nil, status.Error(codes.Unavailable, "draining")
case reply := <-s.replies:
return reply, nil
case <-s.ctx.Done():
return nil, s.ctx.Err()
}
}

// A stream whose Send reported its end still holds the status that says whether its
// server refused. A record it hands back waits for that. Sooner, the record would go
// back to the same stream until it ran out of attempts, or ask for the same server
// again as the one with the fewest streams.
func TestEndedStreamHandsItsRecordBackOnceItsRefusalCounts(t *testing.T) {
const draining, healthy = "10.0.0.1:1", "10.0.0.2:1"
servers := &endpoints{resolved: []string{draining, healthy}, ready: []string{draining, healthy}}
hold := make(chan struct{})
rejected := newScriptedStream(hold)
client := &scriptedStreamsClient{streams: []*scriptedStream{rejected, newScriptedStream(nil)}}
pool := newLanePool(client, servers)
t.Cleanup(pool.close)
command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil)
if err != nil {
t.Fatal(err)
}
// A busy stream on the healthy server makes the group want another stream, on
// the server with the fewest.
key := laneKey{store: "records"}
busy := &lane{address: healthy, expires: time.Now().Add(laneLifetime)}
busy.load.Store(int64(laneOpenAt))
group := &laneGroup{lanes: []*lane{busy}}
pool.groups[key] = group
type finished struct {
event *pb.Event
shared bool
err error
}
done := make(chan finished, 1)
go func() {
event, shared, err := pool.finish(t.Context(), key, command, nil)
result := finished{event: event, shared: shared, err: err}
done <- result
}()
<-rejected.sending
select {
case result := <-done:
t.Fatal("record left a stream whose receiver had not run", result)
case <-time.After(50 * time.Millisecond):
}
close(hold)
result := <-done
if result.err != nil || !result.shared || result.event.GetMutationResult().GetOutcome() != MutationApplied {
t.Fatal("record did not move to a shared stream", result)
}
if asked := client.servers(); !slices.Equal(asked, []string{draining, healthy}) {
t.Fatal("record asked for the server that refused it", asked)
}
}

// A record stream that failed joins its writers, one of which can be waiting for
// the receiver of a stream that ended. Canceling the stream ends that wait.
func TestFailedRecordStreamJoinsAWriterWaitingForItsReceiver(t *testing.T) {
rejected := newScriptedStream(make(chan struct{}))
client := &scriptedStreamsClient{streams: []*scriptedStream{rejected}}
pool := newLanePool(client, nil)
t.Cleanup(pool.close)
command, _, err := mutateCommand(putRequest("records/s:next", `{}`), nil)
if err != nil {
t.Fatal(err)
}
key := pool.ownKey("records", false)
queued := pool.enqueue(t.Context(), key, command)
if queued.lane == nil {
t.Fatal("no stream took the record", queued.err)
}
queued.lane.signal()
<-rejected.sending
forgotten := make(chan struct{})
go func() {
pool.forget(key, true)
close(forgotten)
}()
select {
case <-forgotten:
case <-time.After(5 * time.Second):
t.Fatal("a failed record stream hung on a stream that ended")
}
}

// A stream that ends leaves its group. One a server rejects at once must not end
// before the group holds it, or it stays there dead until it expires.
func TestRejectedStreamDoesNotStayInItsGroup(t *testing.T) {
hold := make(chan struct{})
close(hold)
stream := newScriptedStream(hold)
client := &scriptedStreamsClient{streams: []*scriptedStream{stream}}
pool := newLanePool(client, nil)
t.Cleanup(pool.close)
key := laneKey{store: "records"}
group := &laneGroup{opening: make(chan struct{})}
pool.groups[key] = group
opening := laneOpening{key: key, group: group}
opened := make(chan *lane, 1)
// Holding the pool's lock stops open before it can add the stream.
pool.mu.Lock()
go func() {
stream, err := pool.open(t.Context(), opening)
if err != nil {
t.Error(err)
}
opened <- stream
}()
select {
case <-stream.receiving:
t.Error("stream received before its group held it")
case <-time.After(50 * time.Millisecond):
}
pool.mu.Unlock()
// A stream's context ends after it has left its group.
<-(<-opened).ctx.Done()
pool.mu.Lock()
defer pool.mu.Unlock()
if len(group.lanes) != 0 {
t.Fatal("a rejected stream stayed in its group", len(group.lanes))
}
}

// A Client that knows its servers probes one that has not refused; behind one
// address, a probe can still reach the draining server.
func TestRejectedProbeDoesNotHoldCallsOnADrainingServer(t *testing.T) {
Expand Down