From ca9a0677a1deb272283482033cf3c67d20f8af6c Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 14 Apr 2026 16:13:44 +0200 Subject: [PATCH 01/51] feat: pubsub --- pkg/api/api.go | 8 + pkg/api/pubsub.go | 144 ++++++++++++++ pkg/api/router.go | 10 + pkg/node/node.go | 10 + pkg/p2p/libp2p/libp2p.go | 10 +- pkg/pubsub/mode.go | 210 +++++++++++++++++++++ pkg/pubsub/pubsub.go | 399 +++++++++++++++++++++++++++++++++++++++ pkg/pubsub/ws.go | 109 +++++++++++ 8 files changed, 898 insertions(+), 2 deletions(-) create mode 100644 pkg/api/pubsub.go create mode 100644 pkg/pubsub/mode.go create mode 100644 pkg/pubsub/pubsub.go create mode 100644 pkg/pubsub/ws.go diff --git a/pkg/api/api.go b/pkg/api/api.go index acd838a3ff6..54d349b664c 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -40,6 +40,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/postage" "github.com/ethersphere/bee/v2/pkg/postage/postagecontract" "github.com/ethersphere/bee/v2/pkg/pss" + "github.com/ethersphere/bee/v2/pkg/pubsub" "github.com/ethersphere/bee/v2/pkg/resolver" "github.com/ethersphere/bee/v2/pkg/resolver/client/ens" "github.com/ethersphere/bee/v2/pkg/resolver/multiresolver" @@ -93,6 +94,9 @@ const ( SwarmActTimestampHeader = "Swarm-Act-Timestamp" SwarmActPublisherHeader = "Swarm-Act-Publisher" SwarmActHistoryAddressHeader = "Swarm-Act-History-Address" + SwarmPubsubPeerHeader = "Swarm-Pubsub-Peer" + SwarmPubsubGsocPublicKeyHeader = "Swarm-Pubsub-Gsoc-Public-Key" + SwarmPubsubGsocTopicHeader = "Swarm-Pubsub-Gsoc-Topic" ImmutableHeader = "Immutable" GasPriceHeader = "Gas-Price" @@ -185,6 +189,7 @@ type Service struct { topologyDriver topology.Driver p2p p2p.DebugService + pubsubSvc *pubsub.Service accounting accounting.Interface chequebook chequebook.Service pseudosettle settlement.Interface @@ -268,6 +273,7 @@ type ExtraOptions struct { SyncStatus func() (bool, error) NodeStatus *status.Service PinIntegrity PinIntegrity + PubsubService *pubsub.Service } func New( @@ -355,6 +361,7 @@ func (s *Service) Configure(signer crypto.Signer, tracer *tracing.Tracer, o Opti s.lightNodes = e.LightNodes s.pseudosettle = e.Pseudosettle s.blockTime = e.BlockTime + s.pubsubSvc = e.PubsubService s.statusSem = semaphore.NewWeighted(1) s.postageSem = semaphore.NewWeighted(1) @@ -583,6 +590,7 @@ func (s *Service) corsHandler(h http.Handler) http.Handler { SwarmRedundancyStrategyHeader, SwarmRedundancyFallbackModeHeader, SwarmChunkRetrievalTimeoutHeader, SwarmLookAheadBufferSizeHeader, SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader, SwarmOnlyRootChunk, GasPriceHeader, GasLimitHeader, ImmutableHeader, SwarmActHeader, SwarmActTimestampHeader, SwarmActPublisherHeader, SwarmActHistoryAddressHeader, + SwarmPubsubPeerHeader, SwarmPubsubGsocPublicKeyHeader, SwarmPubsubGsocTopicHeader, } allowedHeadersStr := strings.Join(allowedHeaders, ", ") diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go new file mode 100644 index 00000000000..a97ae52b703 --- /dev/null +++ b/pkg/api/pubsub.go @@ -0,0 +1,144 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api + +import ( + "context" + "encoding/hex" + "net/http" + "time" + + "github.com/ethersphere/bee/v2/pkg/jsonhttp" + "github.com/ethersphere/bee/v2/pkg/pubsub" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" + ma "github.com/multiformats/go-multiaddr" +) + +func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { + logger := s.logger.WithName("pubsub").Build() + + paths := struct { + Topic string `map:"topic" validate:"required"` + }{} + if response := s.mapStructure(mux.Vars(r), &paths); response != nil { + response("invalid path params", logger, w) + return + } + + var topicAddr [32]byte + if decoded, err := hex.DecodeString(paths.Topic); err == nil && len(decoded) == swarm.HashSize { + copy(topicAddr[:], decoded) + } else { + h := swarm.NewHasher() + _, _ = h.Write([]byte(paths.Topic)) + copy(topicAddr[:], h.Sum(nil)) + } + + // Required header: underlay multiaddr + peerHeader := r.Header.Get(SwarmPubsubPeerHeader) + if peerHeader == "" { + jsonhttp.BadRequest(w, "missing Swarm-Pubsub-Peer header") + return + } + underlay, err := ma.NewMultiaddr(peerHeader) + if err != nil { + logger.Debug("invalid peer multiaddr", "value", peerHeader, "error", err) + jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Peer header") + return + } + + // Optional headers: GSOC fields for Participant upgrade + var connectOpts pubsub.ConnectOptions + + gsocPubKeyHex := r.Header.Get(SwarmPubsubGsocPublicKeyHeader) + gsocTopicHex := r.Header.Get(SwarmPubsubGsocTopicHeader) + if gsocPubKeyHex != "" && gsocTopicHex != "" { + gsocOwner, err := hex.DecodeString(gsocPubKeyHex) + if err != nil { + jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Gsoc-Public-Key header") + return + } + gsocID, err := hex.DecodeString(gsocTopicHex) + if err != nil { + jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Gsoc-Topic header") + return + } + connectOpts.GsocOwner = gsocOwner + connectOpts.GsocID = gsocID + connectOpts.ReadWrite = true + } + + headers := struct { + KeepAlive time.Duration `map:"Swarm-Keep-Alive"` + }{} + if response := s.mapStructure(r.Header, &headers); response != nil { + response("invalid header params", logger, w) + return + } + + if s.beeMode == DevMode { + logger.Warning("pubsub endpoint is disabled in dev mode") + jsonhttp.BadRequest(w, errUnsupportedDevNodeOperation) + return + } + + // Connect to broker peer + ctx, cancel := context.WithCancel(context.Background()) + subscriberConn, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) + if err != nil { + cancel() + logger.Debug("pubsub connect failed", "error", err) + jsonhttp.InternalServerError(w, "pubsub connect failed") + return + } + + // Upgrade to WebSocket + upgrader := websocket.Upgrader{ + ReadBufferSize: swarm.ChunkWithSpanSize, + WriteBufferSize: swarm.ChunkWithSpanSize, + CheckOrigin: s.checkOrigin, + } + + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + cancel() + _ = subscriberConn.Stream.Close() + logger.Debug("websocket upgrade failed", "error", err) + logger.Error(nil, "websocket upgrade failed") + jsonhttp.InternalServerError(w, "upgrade failed") + return + } + + pingPeriod := headers.KeepAlive * time.Second + if pingPeriod == 0 { + pingPeriod = time.Minute + } + + isParticipant := connectOpts.ReadWrite + + s.wsWg.Add(1) + go func() { + pubsub.ListeningWs(ctx, conn, pubsub.WsOptions{PingPeriod: pingPeriod, Cancel: cancel}, logger, subscriberConn, isParticipant) + _ = conn.Close() + subscriberConn.Cancel() + s.wsWg.Done() + }() +} + +func (s *Service) pubsubListHandler(w http.ResponseWriter, r *http.Request) { + if s.pubsubSvc == nil { + jsonhttp.NotFound(w, "pubsub service not available") + return + } + + topics := s.pubsubSvc.Topics() + jsonhttp.OK(w, struct { + Topics []pubsub.TopicInfo `json:"topics"` + }{ + Topics: topics, + }) +} diff --git a/pkg/api/router.go b/pkg/api/router.go index 3d61fc98c00..4a3d8594b2c 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -366,6 +366,16 @@ func (s *Service) mountAPI() { ), }) + handle("/pubsub/{topic}", web.ChainHandlers( + web.FinalHandlerFunc(s.pubsubWsHandler), + )) + + handle("/pubsub/", web.ChainHandlers( + web.FinalHandler(jsonhttp.MethodHandler{ + "GET": http.HandlerFunc(s.pubsubListHandler), + }), + )) + handle("/pss/subscribe/{topic}", jsonhttp.MethodHandler{ "GET": http.HandlerFunc(s.pssWsHandler), }) diff --git a/pkg/node/node.go b/pkg/node/node.go index 53d96ec5419..df37a9bb883 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -48,6 +48,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/pricer" "github.com/ethersphere/bee/v2/pkg/pricing" "github.com/ethersphere/bee/v2/pkg/pss" + "github.com/ethersphere/bee/v2/pkg/pubsub" "github.com/ethersphere/bee/v2/pkg/puller" "github.com/ethersphere/bee/v2/pkg/pullsync" "github.com/ethersphere/bee/v2/pkg/pusher" @@ -193,6 +194,8 @@ type Options struct { WarmupTime time.Duration WelcomeMessage string WhitelistedWithdrawalAddress []string + PubsubBrokerMode bool + PubsubMaxConnections int } const ( @@ -669,6 +672,7 @@ func NewBee( Nonce: nonce, ValidateOverlay: chainEnabled, Registry: registry, + PubsubReservedStreamSlots: o.PubsubMaxConnections, }) if err != nil { return nil, fmt.Errorf("p2p service: %w", err) @@ -741,6 +745,11 @@ func NewBee( return nil, fmt.Errorf("init batch service: %w", err) } + pubsubSvc := pubsub.New(p2ps, logger, o.PubsubBrokerMode, o.PubsubMaxConnections) + if err = p2ps.AddProtocol(pubsubSvc.Protocol()); err != nil { + return nil, fmt.Errorf("pubsub protocol: %w", err) + } + // Construct protocols. pingPong := pingpong.New(p2ps, logger, tracer) @@ -1310,6 +1319,7 @@ func NewBee( SyncStatus: syncStatusFn, NodeStatus: nodeStatus, PinIntegrity: localStore.PinIntegrity(), + PubsubService: pubsubSvc, } if o.APIAddr != "" { diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 316cbd6171f..9bdc73143cb 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -149,6 +149,7 @@ type Options struct { HeadersRWTimeout time.Duration Registry *prometheus.Registry autoTLSCertManager autoTLSCertManager + PubsubReservedStreamSlots int } func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay swarm.Address, addr string, ab addressbook.Putter, storer storage.StateStorer, lightNodes *lightnode.Container, logger log.Logger, tracer *tracing.Tracer, o Options) (s *Service, returnErr error) { @@ -209,11 +210,16 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay } // Tweak certain settings + inboundLimit := rcmgr.LimitVal(IncomingStreamCountLimit - o.PubsubReservedStreamSlots) + if inboundLimit < 0 { + inboundLimit = 0 + } + cfg := rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{ - Streams: IncomingStreamCountLimit + OutgoingStreamCountLimit, + Streams: inboundLimit + OutgoingStreamCountLimit, StreamsOutbound: OutgoingStreamCountLimit, - StreamsInbound: IncomingStreamCountLimit, + StreamsInbound: inboundLimit, }, } diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go new file mode 100644 index 00000000000..9d0d3d0663d --- /dev/null +++ b/pkg/pubsub/mode.go @@ -0,0 +1,210 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pubsub + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +var ErrInvalidSignature = errors.New("pubsub: invalid SOC signature") + +const ( + // P2P headers + HeaderGsocOwner = "pubsub-gsoc-owner" + HeaderGsocID = "pubsub-gsoc-id" +) + +// Mode defines mode-specific behavior for the pubsub protocol. +// Each mode determines its own roles, wire format, and message handling. +type Mode interface { + ID() uint8 + TopicAddress() swarm.Address + Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) + + // Broker side - participant + ValidateParticipant(bc *brokerConn, headers p2p.Headers) error + ReadParticipantMessage(stream p2p.Stream) ([]byte, error) + + // Broker side - broadcast + FormatBroadcast(bt *brokerConn, sub *brokerSubscriber, rawMsg []byte) []byte + + // Subscriber/WS side + ReadBrokerMessage(stream p2p.Stream) ([]byte, error) +} + +// --- GSOC Ephemeral Mode (mode 1) --- + +const ( + // Message types (Broker → Subscriber) + MsgTypeHandshake byte = 0x01 + MsgTypeData byte = 0x02 +) + +// GSOCEphemeralMode implements Mode for GSOC ephemeral messaging. +type GSOCEphemeralMode struct { + topicAddress swarm.Address + gsocOwner []byte + gsocID []byte + handshakeHappened bool // true after the first message from broker. sends validation metadata +} + +var _ Mode = (*GSOCEphemeralMode)(nil) + +func NewGSOCEphemeralMode(topicAddress []byte) *GSOCEphemeralMode { + return &GSOCEphemeralMode{ + topicAddress: swarm.NewAddress(topicAddress), + } +} + +func (m *GSOCEphemeralMode) ID() uint8 { return ModeGSOCEphemeral } + +func (m *GSOCEphemeralMode) TopicAddress() swarm.Address { return m.topicAddress.Clone() } + +func (m *GSOCEphemeralMode) Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) { + var rw byte + if opts.ReadWrite { + rw = 1 + } + headers := p2p.Headers{ + HeaderTopicAddress: m.topicAddress.Bytes(), + HeaderMode: {m.ID()}, + HeaderReadWrite: {rw}, + } + if len(opts.GsocOwner) > 0 { + headers[HeaderGsocOwner] = opts.GsocOwner + } + if len(opts.GsocID) > 0 { + headers[HeaderGsocID] = opts.GsocID + } + return p.NewStream(ctx, overlay, headers, protocolName, protocolVersion, streamName) +} + +// ValidateParticipant sets SOC parameters on the broker side so it can validate the messages. +func (m *GSOCEphemeralMode) ValidateParticipant(bc *brokerConn, headers p2p.Headers) error { + gsocOwner := headers[HeaderGsocOwner] + gsocID := headers[HeaderGsocID] + + bc.mu.Lock() + m.setGsocParams(gsocOwner, gsocID) + set := m.gsocID != nil + bc.mu.Unlock() + + if !set { + return ErrWrongHeaders + } + return nil +} + +// FormatBroadcast formats a raw participant message for delivery to a subscriber. +// First delivery to each subscriber includes a handshake with SOC identity; subsequent are data-only. +func (m *GSOCEphemeralMode) FormatBroadcast(bt *brokerConn, sub *brokerSubscriber, rawMsg []byte) []byte { + if !m.handshakeHappened { + // Handshake: [1B type=0x01][32B SOC ID][20B owner][65B sig][4B span][NB payload] + msg := make([]byte, 1+IDSize+OwnerSize+len(rawMsg)) + msg[0] = MsgTypeHandshake + copy(msg[1:1+IDSize], m.gsocID) + copy(msg[1+IDSize:1+IDSize+OwnerSize], m.gsocOwner) + copy(msg[1+IDSize+OwnerSize:], rawMsg) + m.handshakeHappened = true + return msg + } + + // Data: [1B type=0x02][65B sig][4B span][NB payload] + msg := make([]byte, 1+len(rawMsg)) + msg[0] = MsgTypeData + copy(msg[1:], rawMsg) + return msg +} + +// ReadParticipantMessage reads [65B sig][4B span][NB payload (max 4KB)] from the stream, +// constructs and validates the SOC chunk and returns that. +func (m *GSOCEphemeralMode) ReadParticipantMessage(stream p2p.Stream) ([]byte, error) { + sig := make([]byte, SigSize) + if _, err := io.ReadFull(stream, sig); err != nil { + return nil, err + } + spanBytes := make([]byte, SpanSize) + if _, err := io.ReadFull(stream, spanBytes); err != nil { + return nil, err + } + span := min(binary.LittleEndian.Uint32(spanBytes), MaxPayload) + + payload := make([]byte, span) + if _, err := io.ReadFull(stream, payload); err != nil { + return nil, err + } + + // Construct SOC chunk with the known topic address: [ID (32B)][sig (65B)][span (4B)][payload] + // and validate whether message is valid + socData := make([]byte, IDSize+SigSize+SpanSize+int(span)) + copy(socData, m.gsocID) + copy(socData[IDSize:], sig) + copy(socData[IDSize+SigSize:], spanBytes) + copy(socData[IDSize+SigSize+SpanSize:], payload) + + if !soc.Valid(swarm.NewChunk(m.topicAddress, socData)) { + return nil, ErrInvalidSignature + } + + return socData[IDSize:], nil +} + +// ReadBrokerMessage reads one broker→subscriber message and verifies it +func (m *GSOCEphemeralMode) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) { + typeBuf := make([]byte, 1) + if _, err := io.ReadFull(stream, typeBuf); err != nil { + return nil, err + } + + switch typeBuf[0] { + case MsgTypeHandshake: + socID := make([]byte, IDSize) + if _, err := io.ReadFull(stream, socID); err != nil { + return nil, fmt.Errorf("read SOC ID: %w", err) + } + ownerAddr := make([]byte, OwnerSize) + if _, err := io.ReadFull(stream, ownerAddr); err != nil { + return nil, fmt.Errorf("read owner addr: %w", err) + } + m.setGsocParams(ownerAddr, socID) + + return m.ReadParticipantMessage(stream) // same as participant message at this point + + case MsgTypeData: + if m.gsocID == nil { + return nil, fmt.Errorf("pubsub: data message before handshake") + } + return m.ReadParticipantMessage(stream) + + default: + return nil, fmt.Errorf("pubsub: unknown message type: 0x%02x", typeBuf[0]) + } +} + +// setGsocParams sets the GSOC recurring parameters so that messages don't need to include them. +func (m *GSOCEphemeralMode) setGsocParams(gsocOwner, gsocID []byte) { + if m.gsocOwner != nil { + return + } + // Verify got socId and address match with topicaddress + addr, err := soc.CreateAddress(gsocID, gsocOwner) + if err != nil || !bytes.Equal(addr.Bytes(), m.topicAddress.Bytes()) { + return + } + + m.gsocOwner = make([]byte, OwnerSize) + copy(m.gsocOwner, gsocOwner) + m.gsocID = make([]byte, IDSize) + copy(m.gsocID, gsocID) +} diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go new file mode 100644 index 00000000000..98ce8aa4867 --- /dev/null +++ b/pkg/pubsub/pubsub.go @@ -0,0 +1,399 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pubsub + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/swarm" + ma "github.com/multiformats/go-multiaddr" +) + +const ( + loggerName = "pubsub" + protocolName = "pubsub" + protocolVersion = "1.0.0" + streamName = "msg" + + // p2p stream header keys + HeaderTopicAddress = "pubsub-topic-address" + HeaderMode = "pubsub-mode" + HeaderReadWrite = "pubsub-readwrite" // 1 = read+write (participant), 0 = read-only (subscriber) + + // Mode constants + ModeGSOCEphemeral uint8 = 1 + + // Wire format sizes + SpanSize = 4 // pubsub span: uint32 little-endian + MaxPayload = swarm.ChunkSize + SigSize = swarm.SocSignatureSize + IDSize = swarm.HashSize + OwnerSize = crypto.AddressSize +) + +var ( + ErrBrokerDisabled = errors.New("pubsub: broker mode is disabled") + ErrMaxConnections = errors.New("pubsub: max connections reached") + ErrInvalidHandshake = errors.New("pubsub: handshake verification failed") + ErrWrongHeaders = errors.New("pubsub: wrong required headers") + ErrTopicMismatch = errors.New("pubsub: topic address mismatch") +) + +func newMode(topicAddr [32]byte, modeID uint8) (Mode, error) { + switch modeID { + case ModeGSOCEphemeral: + return NewGSOCEphemeralMode(topicAddr[:]), nil + default: + return nil, fmt.Errorf("pubsub: unknown mode: %d", modeID) + } +} + +// ConnectOptions carries optional mode-specific parameters for Connect. +type ConnectOptions struct { + ReadWrite bool // true = participant (read+write), false = subscriber (read-only) + GsocOwner []byte + GsocID []byte +} + +// TopicInfo describes a topic for the list endpoint. +type TopicInfo struct { + TopicAddress string `json:"topicAddress"` + Mode uint8 `json:"mode"` + Role string `json:"role"` + Connections []string `json:"connections"` +} + +// brokerSubscriber holds a subscriber's stream and outgoing message channel. +type brokerSubscriber struct { + overlay swarm.Address + stream p2p.Stream + outCh chan []byte + cancel context.CancelFunc +} + +// brokerConn manages all connections for a single topic on the broker side. +type brokerConn struct { + mu sync.RWMutex + mode Mode + subscribers map[string]*brokerSubscriber +} + +// SubscriberConn represents the subscriber-side connection to a broker. +type SubscriberConn struct { + Stream p2p.Stream + TopicAddr [32]byte + Mode Mode + Overlay swarm.Address + Cancel context.CancelFunc +} + +// P2P groups the p2p capabilities needed by the pubsub service. +type P2P interface { + p2p.Service + p2p.Streamer +} + +// Service is the pubsub protocol service. +type Service struct { + mu sync.RWMutex + p2p P2P + logger log.Logger + brokerMode bool + maxConns int + brokerConns map[[32]byte]*brokerConn // topic address -> broker connection + subscriberConns map[[32]byte]*SubscriberConn // topic address -> subscriber connection +} + +func New(p2p P2P, logger log.Logger, brokerMode bool, maxConns int) *Service { + s := &Service{ + p2p: p2p, + logger: logger.WithName(loggerName).Register(), + brokerMode: brokerMode, + maxConns: maxConns, + brokerConns: make(map[[32]byte]*brokerConn), + subscriberConns: make(map[[32]byte]*SubscriberConn), + } + return s +} + +// Protocol returns the p2p protocol spec. +func (s *Service) Protocol() p2p.ProtocolSpec { + return p2p.ProtocolSpec{ + Name: protocolName, + Version: protocolVersion, + StreamSpecs: []p2p.StreamSpec{ + { + Name: streamName, + Handler: s.brokerHandler, + }, + }, + } +} + +// Connect establishes a subscriber connection to a broker peer. +func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr [32]byte, modeID uint8, opts ConnectOptions) (*SubscriberConn, error) { + m, err := newMode(topicAddr, modeID) + if err != nil { + return nil, err + } + + bzzAddr, err := s.p2p.Connect(ctx, []ma.Multiaddr{underlay}) + if err != nil && !errors.Is(err, p2p.ErrAlreadyConnected) { + return nil, fmt.Errorf("connect to peer: %w", err) + } + + stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) + if err != nil { + return nil, fmt.Errorf("open stream: %w", err) + } + + connCtx, cancel := context.WithCancel(ctx) + sc := &SubscriberConn{ + Stream: stream, + TopicAddr: topicAddr, + Mode: m, + Overlay: bzzAddr.Overlay, + Cancel: cancel, + } + + s.mu.Lock() + s.subscriberConns[topicAddr] = sc + s.mu.Unlock() + + go func() { + <-connCtx.Done() + s.mu.Lock() + delete(s.subscriberConns, topicAddr) + s.mu.Unlock() + _ = stream.FullClose() + }() + + return sc, nil +} + +// Topics returns info about all active topics. +func (s *Service) Topics() []TopicInfo { + s.mu.RLock() + defer s.mu.RUnlock() + + var topics []TopicInfo + + for addr, bt := range s.brokerConns { + bt.mu.RLock() + conns := make([]string, 0, len(bt.subscribers)) + for _, sub := range bt.subscribers { + conns = append(conns, sub.overlay.String()) + } + bt.mu.RUnlock() + topics = append(topics, TopicInfo{ + TopicAddress: fmt.Sprintf("%x", addr), + Mode: bt.mode.ID(), + Role: "broker", + Connections: conns, + }) + } + + for addr, sc := range s.subscriberConns { + topics = append(topics, TopicInfo{ + TopicAddress: fmt.Sprintf("%x", addr), + Mode: sc.Mode.ID(), + Role: "subscriber", + Connections: []string{sc.Overlay.String()}, + }) + } + + return topics +} + +// brokerHandler handles incoming streams on the broker side. +func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.Stream) error { + if !s.brokerMode { + _ = stream.Reset() + return ErrBrokerDisabled + } + + headers := stream.Headers() + + topicAddrBytes := headers[HeaderTopicAddress] + if len(topicAddrBytes) != IDSize { + _ = stream.Reset() + return ErrWrongHeaders + } + var topicAddr [32]byte + copy(topicAddr[:], topicAddrBytes) + + modeBytes := headers[HeaderMode] + if len(modeBytes) != 1 { + _ = stream.Reset() + return ErrWrongHeaders + } + bc, err := s.getOrCreateBrokerConn(topicAddr, modeBytes[0]) + if err != nil { + _ = stream.Reset() + return err + } + + rwBytes := headers[HeaderReadWrite] + if len(rwBytes) != 1 { + _ = stream.Reset() + return ErrWrongHeaders + } + if rwBytes[0] == 1 { + return s.handleParticipant(ctx, peer, stream, bc, headers) + } + return s.handleSubscriber(ctx, peer, stream, bc) +} + +// registerBrokerConn registers the peer as a broadcast recipient on bc, starts a write goroutine, +// and returns the connection context, its cancel func, and an unregister func to call on exit. +func (s *Service) registerBrokerConn(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn) (connCtx context.Context, cancel context.CancelFunc, unregister func()) { + connCtx, cancel = context.WithCancel(ctx) + + conn := &brokerSubscriber{ + overlay: peer.Address, + stream: stream, + outCh: make(chan []byte, 256), + cancel: cancel, + } + + overlayKey := peer.Address.String() + bc.mu.Lock() + bc.subscribers[overlayKey] = conn + bc.mu.Unlock() + + go func() { + for { + select { + case <-connCtx.Done(): + return + case msg := <-conn.outCh: + if err := writeRaw(stream, msg); err != nil { + cancel() + return + } + } + } + }() + + unregister = func() { + bc.mu.Lock() + delete(bc.subscribers, overlayKey) + bc.mu.Unlock() + } + return +} + +func (s *Service) handleSubscriber(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn) error { + bc.mu.RLock() + connCount := len(bc.subscribers) + bc.mu.RUnlock() + if s.maxConns > 0 && connCount >= s.maxConns { + _ = stream.Reset() + return ErrMaxConnections + } + + subCtx, cancel, unregister := s.registerBrokerConn(ctx, peer, stream, bc) + defer cancel() + defer unregister() + + s.logger.Debug("subscriber connected", "peer", peer.Address, "topic", bc.mode.TopicAddress()) + + <-subCtx.Done() + if errors.Is(subCtx.Err(), context.Canceled) { + return nil + } + return subCtx.Err() +} + +func (s *Service) handleParticipant(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn, headers p2p.Headers) error { + if err := bc.mode.ValidateParticipant(bc, headers); err != nil { + _ = stream.Reset() + return err + } + + partCtx, cancel, unregister := s.registerBrokerConn(ctx, peer, stream, bc) + defer cancel() + defer unregister() + + s.logger.Debug("participant connected", "peer", peer.Address, "topic", bc.mode.TopicAddress()) + + for { + select { + case <-partCtx.Done(): + if errors.Is(partCtx.Err(), context.Canceled) { + return nil + } + return partCtx.Err() + default: + } + + rawMsg, err := bc.mode.ReadParticipantMessage(stream) + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("read participant message: %w", err) + } + + s.broadcastToSubscribers(bc, rawMsg) + } +} + +// broadcastToSubscribers sends a message to all subscribers of a topic. +func (s *Service) broadcastToSubscribers(bc *brokerConn, rawMsg []byte) { + bc.mu.Lock() + defer bc.mu.Unlock() + + for _, sub := range bc.subscribers { + msg := bc.mode.FormatBroadcast(bc, sub, rawMsg) + + select { + case sub.outCh <- msg: + default: + s.logger.Warning("subscriber message queue full, dropping message", "peer", sub.overlay) + } + } +} + +func (s *Service) getOrCreateBrokerConn(topicAddr [32]byte, modeID uint8) (*brokerConn, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if bt, ok := s.brokerConns[topicAddr]; ok { + return bt, nil + } + + m, err := newMode(topicAddr, modeID) + if err != nil { + return nil, err + } + + bc := &brokerConn{ + mode: m, + subscribers: make(map[string]*brokerSubscriber), + } + s.brokerConns[topicAddr] = bc + return bc, nil +} + +// writeRaw writes raw bytes to the stream. +func writeRaw(stream p2p.Stream, data []byte) error { + c := 0 + for c < len(data) { + n, err := stream.Write(data[c:]) + if err != nil { + return err + } + c += n + } + return nil +} diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go new file mode 100644 index 00000000000..1a19967a88d --- /dev/null +++ b/pkg/pubsub/ws.go @@ -0,0 +1,109 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pubsub + +import ( + "context" + "fmt" + "time" + + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/gorilla/websocket" +) + +type WsOptions struct { + PingPeriod time.Duration + Cancel context.CancelFunc +} + +// ListeningWs bridges a subscriber's p2p stream to a WebSocket connection. +// The Mode on sc.Mode handles all wire-format details: reading broker messages, +// verifying them, and returning the payload to forward to the WebSocket. +// If the subscriber is a Participant, it also reads from the WebSocket +// and writes raw messages to the p2p stream. +func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, logger log.Logger, sc *SubscriberConn, isParticipant bool) { + var ( + ticker = time.NewTicker(options.PingPeriod) + writeDeadline = options.PingPeriod + time.Second + readDeadline = options.PingPeriod + time.Second + ) + + conn.SetCloseHandler(func(code int, text string) error { + logger.Debug("pubsub ws: client gone", "topic", fmt.Sprintf("%x", sc.TopicAddr), "code", code, "message", text) + return nil + }) + + // If Participant, read from WebSocket and write to p2p stream (send to Broker). + if isParticipant { + go func() { + for { + if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil { + logger.Debug("pubsub ws: set read deadline failed", "error", err) + break + } + _, p, err := conn.ReadMessage() + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { + logger.Debug("pubsub ws: read error", "error", err) + } + break + } + + if err := writeRaw(sc.Stream, p); err != nil { + logger.Debug("pubsub ws: write to p2p stream failed", "error", err) + break + } + } + options.Cancel() + }() + } + + // Read from p2p stream (Broker messages) and forward to WebSocket. + go func() { + for { + select { + case <-ctx.Done(): + return + default: + } + + wsPayload, err := sc.Mode.ReadBrokerMessage(sc.Stream) + if err != nil { + if ctx.Err() == nil { + logger.Debug("pubsub ws: read broker message failed", "error", err) + } + options.Cancel() + return + } + + if err := conn.WriteMessage(websocket.BinaryMessage, wsPayload); err != nil { + logger.Debug("pubsub ws: write to ws failed", "error", err) + options.Cancel() + return + } + } + }() + + defer func() { + ticker.Stop() + _ = conn.Close() + }() + + for { + if err := conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil { + logger.Debug("pubsub ws: set write deadline failed", "error", err) + return + } + select { + case <-ctx.Done(): + _ = conn.WriteMessage(websocket.CloseMessage, []byte{}) + return + case <-ticker.C: + if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} From 218cd080c0f3d48d4175995362fd1d102662e0c8 Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 16 Apr 2026 09:44:21 +0200 Subject: [PATCH 02/51] refactor: mode id enum type --- pkg/pubsub/mode.go | 9 ++++++--- pkg/pubsub/pubsub.go | 12 ++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 9d0d3d0663d..02504890084 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -25,10 +25,13 @@ const ( HeaderGsocID = "pubsub-gsoc-id" ) +// ModeID identifies a pubsub mode. +type ModeID uint8 + // Mode defines mode-specific behavior for the pubsub protocol. // Each mode determines its own roles, wire format, and message handling. type Mode interface { - ID() uint8 + ID() ModeID TopicAddress() swarm.Address Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) @@ -67,7 +70,7 @@ func NewGSOCEphemeralMode(topicAddress []byte) *GSOCEphemeralMode { } } -func (m *GSOCEphemeralMode) ID() uint8 { return ModeGSOCEphemeral } +func (m *GSOCEphemeralMode) ID() ModeID { return ModeGSOCEphemeral } func (m *GSOCEphemeralMode) TopicAddress() swarm.Address { return m.topicAddress.Clone() } @@ -78,7 +81,7 @@ func (m *GSOCEphemeralMode) Connect(ctx context.Context, p p2p.Streamer, overlay } headers := p2p.Headers{ HeaderTopicAddress: m.topicAddress.Bytes(), - HeaderMode: {m.ID()}, + HeaderMode: {byte(m.ID())}, HeaderReadWrite: {rw}, } if len(opts.GsocOwner) > 0 { diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 98ce8aa4867..ab0557454e1 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -30,7 +30,7 @@ const ( HeaderReadWrite = "pubsub-readwrite" // 1 = read+write (participant), 0 = read-only (subscriber) // Mode constants - ModeGSOCEphemeral uint8 = 1 + ModeGSOCEphemeral ModeID = 1 // Wire format sizes SpanSize = 4 // pubsub span: uint32 little-endian @@ -48,7 +48,7 @@ var ( ErrTopicMismatch = errors.New("pubsub: topic address mismatch") ) -func newMode(topicAddr [32]byte, modeID uint8) (Mode, error) { +func newMode(topicAddr [32]byte, modeID ModeID) (Mode, error) { switch modeID { case ModeGSOCEphemeral: return NewGSOCEphemeralMode(topicAddr[:]), nil @@ -67,7 +67,7 @@ type ConnectOptions struct { // TopicInfo describes a topic for the list endpoint. type TopicInfo struct { TopicAddress string `json:"topicAddress"` - Mode uint8 `json:"mode"` + Mode ModeID `json:"mode"` Role string `json:"role"` Connections []string `json:"connections"` } @@ -140,7 +140,7 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } // Connect establishes a subscriber connection to a broker peer. -func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr [32]byte, modeID uint8, opts ConnectOptions) (*SubscriberConn, error) { +func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr [32]byte, modeID ModeID, opts ConnectOptions) (*SubscriberConn, error) { m, err := newMode(topicAddr, modeID) if err != nil { return nil, err @@ -236,7 +236,7 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S _ = stream.Reset() return ErrWrongHeaders } - bc, err := s.getOrCreateBrokerConn(topicAddr, modeBytes[0]) + bc, err := s.getOrCreateBrokerConn(topicAddr, ModeID(modeBytes[0])) if err != nil { _ = stream.Reset() return err @@ -364,7 +364,7 @@ func (s *Service) broadcastToSubscribers(bc *brokerConn, rawMsg []byte) { } } -func (s *Service) getOrCreateBrokerConn(topicAddr [32]byte, modeID uint8) (*brokerConn, error) { +func (s *Service) getOrCreateBrokerConn(topicAddr [32]byte, modeID ModeID) (*brokerConn, error) { s.mu.Lock() defer s.mu.Unlock() From 2f1f4dc6c8c353789fcc9fcc73dd0ed1edae1e8e Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 16 Apr 2026 09:48:00 +0200 Subject: [PATCH 03/51] refactor: rename participant to publisher --- pkg/api/pubsub.go | 6 +++--- pkg/pubsub/mode.go | 20 ++++++++++---------- pkg/pubsub/pubsub.go | 16 ++++++++-------- pkg/pubsub/ws.go | 8 ++++---- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index a97ae52b703..edf71e236e4 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -51,7 +51,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { return } - // Optional headers: GSOC fields for Participant upgrade + // Optional headers: GSOC fields for Publisher upgrade var connectOpts pubsub.ConnectOptions gsocPubKeyHex := r.Header.Get(SwarmPubsubGsocPublicKeyHeader) @@ -118,11 +118,11 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { pingPeriod = time.Minute } - isParticipant := connectOpts.ReadWrite + isPublisher := connectOpts.ReadWrite s.wsWg.Add(1) go func() { - pubsub.ListeningWs(ctx, conn, pubsub.WsOptions{PingPeriod: pingPeriod, Cancel: cancel}, logger, subscriberConn, isParticipant) + pubsub.ListeningWs(ctx, conn, pubsub.WsOptions{PingPeriod: pingPeriod, Cancel: cancel}, logger, subscriberConn, isPublisher) _ = conn.Close() subscriberConn.Cancel() s.wsWg.Done() diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 02504890084..0d74777c470 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -35,9 +35,9 @@ type Mode interface { TopicAddress() swarm.Address Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) - // Broker side - participant - ValidateParticipant(bc *brokerConn, headers p2p.Headers) error - ReadParticipantMessage(stream p2p.Stream) ([]byte, error) + // Broker side - publisher + ValidatePublisher(bc *brokerConn, headers p2p.Headers) error + ReadPublisherMessage(stream p2p.Stream) ([]byte, error) // Broker side - broadcast FormatBroadcast(bt *brokerConn, sub *brokerSubscriber, rawMsg []byte) []byte @@ -93,8 +93,8 @@ func (m *GSOCEphemeralMode) Connect(ctx context.Context, p p2p.Streamer, overlay return p.NewStream(ctx, overlay, headers, protocolName, protocolVersion, streamName) } -// ValidateParticipant sets SOC parameters on the broker side so it can validate the messages. -func (m *GSOCEphemeralMode) ValidateParticipant(bc *brokerConn, headers p2p.Headers) error { +// ValidatePublisher sets SOC parameters on the broker side so it can validate the messages. +func (m *GSOCEphemeralMode) ValidatePublisher(bc *brokerConn, headers p2p.Headers) error { gsocOwner := headers[HeaderGsocOwner] gsocID := headers[HeaderGsocID] @@ -109,7 +109,7 @@ func (m *GSOCEphemeralMode) ValidateParticipant(bc *brokerConn, headers p2p.Head return nil } -// FormatBroadcast formats a raw participant message for delivery to a subscriber. +// FormatBroadcast formats a raw publisher message for delivery to a subscriber. // First delivery to each subscriber includes a handshake with SOC identity; subsequent are data-only. func (m *GSOCEphemeralMode) FormatBroadcast(bt *brokerConn, sub *brokerSubscriber, rawMsg []byte) []byte { if !m.handshakeHappened { @@ -130,9 +130,9 @@ func (m *GSOCEphemeralMode) FormatBroadcast(bt *brokerConn, sub *brokerSubscribe return msg } -// ReadParticipantMessage reads [65B sig][4B span][NB payload (max 4KB)] from the stream, +// ReadPublisherMessage reads [65B sig][4B span][NB payload (max 4KB)] from the stream, // constructs and validates the SOC chunk and returns that. -func (m *GSOCEphemeralMode) ReadParticipantMessage(stream p2p.Stream) ([]byte, error) { +func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, error) { sig := make([]byte, SigSize) if _, err := io.ReadFull(stream, sig); err != nil { return nil, err @@ -182,13 +182,13 @@ func (m *GSOCEphemeralMode) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) } m.setGsocParams(ownerAddr, socID) - return m.ReadParticipantMessage(stream) // same as participant message at this point + return m.ReadPublisherMessage(stream) // same as publisher message at this point case MsgTypeData: if m.gsocID == nil { return nil, fmt.Errorf("pubsub: data message before handshake") } - return m.ReadParticipantMessage(stream) + return m.ReadPublisherMessage(stream) default: return nil, fmt.Errorf("pubsub: unknown message type: 0x%02x", typeBuf[0]) diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index ab0557454e1..d56bbbf3c76 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -27,7 +27,7 @@ const ( // p2p stream header keys HeaderTopicAddress = "pubsub-topic-address" HeaderMode = "pubsub-mode" - HeaderReadWrite = "pubsub-readwrite" // 1 = read+write (participant), 0 = read-only (subscriber) + HeaderReadWrite = "pubsub-readwrite" // 1 = read+write (publisher), 0 = read-only (subscriber) // Mode constants ModeGSOCEphemeral ModeID = 1 @@ -59,7 +59,7 @@ func newMode(topicAddr [32]byte, modeID ModeID) (Mode, error) { // ConnectOptions carries optional mode-specific parameters for Connect. type ConnectOptions struct { - ReadWrite bool // true = participant (read+write), false = subscriber (read-only) + ReadWrite bool // true = publisher (read+write), false = subscriber (read-only) GsocOwner []byte GsocID []byte } @@ -248,7 +248,7 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S return ErrWrongHeaders } if rwBytes[0] == 1 { - return s.handleParticipant(ctx, peer, stream, bc, headers) + return s.handlePublisher(ctx, peer, stream, bc, headers) } return s.handleSubscriber(ctx, peer, stream, bc) } @@ -314,8 +314,8 @@ func (s *Service) handleSubscriber(ctx context.Context, peer p2p.Peer, stream p2 return subCtx.Err() } -func (s *Service) handleParticipant(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn, headers p2p.Headers) error { - if err := bc.mode.ValidateParticipant(bc, headers); err != nil { +func (s *Service) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn, headers p2p.Headers) error { + if err := bc.mode.ValidatePublisher(bc, headers); err != nil { _ = stream.Reset() return err } @@ -324,7 +324,7 @@ func (s *Service) handleParticipant(ctx context.Context, peer p2p.Peer, stream p defer cancel() defer unregister() - s.logger.Debug("participant connected", "peer", peer.Address, "topic", bc.mode.TopicAddress()) + s.logger.Debug("publisher connected", "peer", peer.Address, "topic", bc.mode.TopicAddress()) for { select { @@ -336,12 +336,12 @@ func (s *Service) handleParticipant(ctx context.Context, peer p2p.Peer, stream p default: } - rawMsg, err := bc.mode.ReadParticipantMessage(stream) + rawMsg, err := bc.mode.ReadPublisherMessage(stream) if err != nil { if errors.Is(err, io.EOF) { return nil } - return fmt.Errorf("read participant message: %w", err) + return fmt.Errorf("read publisher message: %w", err) } s.broadcastToSubscribers(bc, rawMsg) diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index 1a19967a88d..ac6ed49d230 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -21,9 +21,9 @@ type WsOptions struct { // ListeningWs bridges a subscriber's p2p stream to a WebSocket connection. // The Mode on sc.Mode handles all wire-format details: reading broker messages, // verifying them, and returning the payload to forward to the WebSocket. -// If the subscriber is a Participant, it also reads from the WebSocket +// If the subscriber is a Publisher, it also reads from the WebSocket // and writes raw messages to the p2p stream. -func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, logger log.Logger, sc *SubscriberConn, isParticipant bool) { +func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, logger log.Logger, sc *SubscriberConn, isPublisher bool) { var ( ticker = time.NewTicker(options.PingPeriod) writeDeadline = options.PingPeriod + time.Second @@ -35,8 +35,8 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l return nil }) - // If Participant, read from WebSocket and write to p2p stream (send to Broker). - if isParticipant { + // If Publisher, read from WebSocket and write to p2p stream (send to Broker). + if isPublisher { go func() { for { if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil { From 4a01a685a2d3601e9a5562bf08ed506bba9ecb29 Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 16 Apr 2026 15:08:58 +0200 Subject: [PATCH 04/51] docs: openapi --- openapi/Swarm.yaml | 54 ++++++++++++++++++++++++++++++++++++++++ openapi/SwarmCommon.yaml | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 70f9ffaf4be..5a455f19527 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -2541,3 +2541,57 @@ paths: $ref: "SwarmCommon.yaml#/components/responses/400" default: description: Default response. + + "/pubsub/{topic}": + get: + summary: Connect to a pubsub topic via WebSocket + description: | + Opens a WebSocket connection to a pubsub topic. The connection acts as either a publisher (read+write) + or subscriber (read-only) depending on the presence of GSOC headers. + + **WebSocket protocol:** + - Inbound (client → node, publisher only): raw SOC payload `[sig:65B][span:4B][payload:N B]` + - Outbound (node → client): raw SOC payload `[sig:65B][span:4B][payload:N B]` + tags: + - Pubsub + parameters: + - in: path + name: topic + schema: + type: string + required: true + description: Topic identifier (hex-encoded address or arbitrary string to be hashed) + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubPeer" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubGsocPublicKey" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubGsocTopic" + - in: header + name: swarm-keep-alive + schema: + type: integer + required: false + description: WebSocket ping period in seconds (default: 60) + responses: + "101": + description: WebSocket upgrade successful + "400": + $ref: "SwarmCommon.yaml#/components/responses/400" + "500": + $ref: "SwarmCommon.yaml#/components/responses/500" + + "/pubsub/": + get: + summary: List all pubsub topics + description: Returns a list of all active pubsub topics this node is participating in (as broker or subscriber) + tags: + - Pubsub + responses: + "200": + description: List of pubsub topics + content: + application/json: + schema: + $ref: "SwarmCommon.yaml#/components/schemas/PubsubTopicListResponse" + "400": + $ref: "SwarmCommon.yaml#/components/responses/400" + "500": + $ref: "SwarmCommon.yaml#/components/responses/500" diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 303245501f9..1e0c2ca54e2 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1095,6 +1095,31 @@ components: required: false description: "Indicates which feed version was resolved (v1 or v2)" + PubsubTopicInfo: + type: object + properties: + topicAddress: + type: string + description: "Hex-encoded topic address" + mode: + type: integer + description: "Pubsub mode identifier" + role: + type: string + description: "Role of this node: 'broker' or 'subscriber'" + connections: + type: array + items: + type: string + description: "List of connected peer overlays" + + PubsubTopicListResponse: + type: object + properties: + topics: + type: array + items: + $ref: "#/components/schemas/PubsubTopicInfo" parameters: GasPriceParameter: @@ -1308,6 +1333,30 @@ components: required: false description: "ACT history Unix timestamp" + SwarmPubsubPeer: + in: header + name: swarm-pubsub-peer + schema: + type: string + required: true + description: "Multiaddress of the broker peer to connect to for pubsub" + + SwarmPubsubGsocPublicKey: + in: header + name: swarm-pubsub-gsoc-public-key + schema: + $ref: "#/components/schemas/HexString" + required: false + description: "GSOC public key (hex) for publisher role. Required together with swarm-pubsub-gsoc-topic to upgrade to publisher." + + SwarmPubsubGsocTopic: + in: header + name: swarm-pubsub-gsoc-topic + schema: + $ref: "#/components/schemas/HexString" + required: false + description: "GSOC topic identifier (hex) for publisher role. Required together with swarm-pubsub-gsoc-public-key to upgrade to publisher." + responses: "200": description: Success From f1710db39d27e5151b7c8b9ce693fb8563791006 Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 17 Apr 2026 16:32:51 +0200 Subject: [PATCH 05/51] fix: eth address instead of public key on api --- openapi/Swarm.yaml | 2 +- openapi/SwarmCommon.yaml | 8 ++++---- pkg/api/api.go | 4 ++-- pkg/api/pubsub.go | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 5a455f19527..f3652654250 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -2562,7 +2562,7 @@ paths: required: true description: Topic identifier (hex-encoded address or arbitrary string to be hashed) - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubPeer" - - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubGsocPublicKey" + - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubGsocEthAddress" - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPubsubGsocTopic" - in: header name: swarm-keep-alive diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 1e0c2ca54e2..8b816b0613e 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1341,13 +1341,13 @@ components: required: true description: "Multiaddress of the broker peer to connect to for pubsub" - SwarmPubsubGsocPublicKey: + SwarmPubsubGsocEthAddress: in: header - name: swarm-pubsub-gsoc-public-key + name: swarm-pubsub-gsoc-eth-address schema: $ref: "#/components/schemas/HexString" required: false - description: "GSOC public key (hex) for publisher role. Required together with swarm-pubsub-gsoc-topic to upgrade to publisher." + description: "GSOC owner Ethereum address (20 bytes, hex-encoded) for publisher role. Required together with swarm-pubsub-gsoc-topic to upgrade to publisher." SwarmPubsubGsocTopic: in: header @@ -1355,7 +1355,7 @@ components: schema: $ref: "#/components/schemas/HexString" required: false - description: "GSOC topic identifier (hex) for publisher role. Required together with swarm-pubsub-gsoc-public-key to upgrade to publisher." + description: "GSOC topic identifier (hex) for publisher role. Required together with swarm-pubsub-gsoc-eth-address to upgrade to publisher." responses: "200": diff --git a/pkg/api/api.go b/pkg/api/api.go index 54d349b664c..15a5563d1d5 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -95,7 +95,7 @@ const ( SwarmActPublisherHeader = "Swarm-Act-Publisher" SwarmActHistoryAddressHeader = "Swarm-Act-History-Address" SwarmPubsubPeerHeader = "Swarm-Pubsub-Peer" - SwarmPubsubGsocPublicKeyHeader = "Swarm-Pubsub-Gsoc-Public-Key" + SwarmPubsubGsocEthAddressHeader = "Swarm-Pubsub-Gsoc-Eth-Address" SwarmPubsubGsocTopicHeader = "Swarm-Pubsub-Gsoc-Topic" ImmutableHeader = "Immutable" @@ -590,7 +590,7 @@ func (s *Service) corsHandler(h http.Handler) http.Handler { SwarmRedundancyStrategyHeader, SwarmRedundancyFallbackModeHeader, SwarmChunkRetrievalTimeoutHeader, SwarmLookAheadBufferSizeHeader, SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader, SwarmOnlyRootChunk, GasPriceHeader, GasLimitHeader, ImmutableHeader, SwarmActHeader, SwarmActTimestampHeader, SwarmActPublisherHeader, SwarmActHistoryAddressHeader, - SwarmPubsubPeerHeader, SwarmPubsubGsocPublicKeyHeader, SwarmPubsubGsocTopicHeader, + SwarmPubsubPeerHeader, SwarmPubsubGsocEthAddressHeader, SwarmPubsubGsocTopicHeader, } allowedHeadersStr := strings.Join(allowedHeaders, ", ") diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index edf71e236e4..e9a6723646b 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -54,12 +54,12 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { // Optional headers: GSOC fields for Publisher upgrade var connectOpts pubsub.ConnectOptions - gsocPubKeyHex := r.Header.Get(SwarmPubsubGsocPublicKeyHeader) + gsocEthAddrHex := r.Header.Get(SwarmPubsubGsocEthAddressHeader) gsocTopicHex := r.Header.Get(SwarmPubsubGsocTopicHeader) - if gsocPubKeyHex != "" && gsocTopicHex != "" { - gsocOwner, err := hex.DecodeString(gsocPubKeyHex) + if gsocEthAddrHex != "" && gsocTopicHex != "" { + gsocOwner, err := hex.DecodeString(gsocEthAddrHex) if err != nil { - jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Gsoc-Public-Key header") + jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Gsoc-Eth-Address header") return } gsocID, err := hex.DecodeString(gsocTopicHex) From 0f05391e453be669696a8ae024e14d3477b84697 Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Mon, 20 Apr 2026 15:07:15 +0200 Subject: [PATCH 06/51] fix: pubsub header parsing --- pkg/api/pubsub.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index e9a6723646b..7a193121684 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -38,8 +38,11 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { copy(topicAddr[:], h.Sum(nil)) } - // Required header: underlay multiaddr + // Required: underlay multiaddr — accept from header or query param (browsers cannot set WS headers) peerHeader := r.Header.Get(SwarmPubsubPeerHeader) + if peerHeader == "" { + peerHeader = r.URL.Query().Get("swarm-pubsub-peer") + } if peerHeader == "" { jsonhttp.BadRequest(w, "missing Swarm-Pubsub-Peer header") return @@ -51,11 +54,17 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { return } - // Optional headers: GSOC fields for Publisher upgrade + // Optional: GSOC fields for Publisher upgrade — accept from header or query param var connectOpts pubsub.ConnectOptions gsocEthAddrHex := r.Header.Get(SwarmPubsubGsocEthAddressHeader) + if gsocEthAddrHex == "" { + gsocEthAddrHex = r.URL.Query().Get("swarm-pubsub-gsoc-eth-address") + } gsocTopicHex := r.Header.Get(SwarmPubsubGsocTopicHeader) + if gsocTopicHex == "" { + gsocTopicHex = r.URL.Query().Get("swarm-pubsub-gsoc-topic") + } if gsocEthAddrHex != "" && gsocTopicHex != "" { gsocOwner, err := hex.DecodeString(gsocEthAddrHex) if err != nil { From c3921c489c36952df60b9753ee700a464787e12d Mon Sep 17 00:00:00 2001 From: nugaon Date: Mon, 20 Apr 2026 16:50:25 +0200 Subject: [PATCH 07/51] fix: config cli option for broker mode --- cmd/bee/cmd/cmd.go | 4 ++++ cmd/bee/cmd/start.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index b4ece436d8d..4fe64f60e86 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -88,6 +88,8 @@ const ( optionAutoTLSDomain = "autotls-domain" optionAutoTLSRegistrationEndpoint = "autotls-registration-endpoint" optionAutoTLSCAEndpoint = "autotls-ca-endpoint" + optionNamePubsubBrokerMode = "pubsub-broker-mode" + optionNamePubsubMaxConnections = "pubsub-max-connections" // blockchain-rpc optionNameBlockchainRpcEndpoint = "blockchain-rpc-endpoint" @@ -334,6 +336,8 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().String(optionAutoTLSDomain, p2pforge.DefaultForgeDomain, "autotls domain") cmd.Flags().String(optionAutoTLSRegistrationEndpoint, p2pforge.DefaultForgeEndpoint, "autotls registration endpoint") cmd.Flags().String(optionAutoTLSCAEndpoint, p2pforge.DefaultCAEndpoint, "autotls certificate authority endpoint") + cmd.Flags().Bool(optionNamePubsubBrokerMode, true, "enable pubsub broker mode") + cmd.Flags().Int(optionNamePubsubMaxConnections, 0, "max pubsub connections per topic (0 = unlimited)") } // preRun must be called from every command's PreRunE, after which c.logger is diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index 5773c4af3e3..7a09256c3fe 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -332,6 +332,8 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo WarmupTime: c.config.GetDuration(optionWarmUpTime), WelcomeMessage: c.config.GetString(optionWelcomeMessage), WhitelistedWithdrawalAddress: c.config.GetStringSlice(optionNameWhitelistedWithdrawalAddress), + PubsubBrokerMode: c.config.GetBool(optionNamePubsubBrokerMode), + PubsubMaxConnections: c.config.GetInt(optionNamePubsubMaxConnections), }) return b, err From 837730f869f59b1d244d57b16f300a4c5d7a53fb Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 11:12:47 +0200 Subject: [PATCH 08/51] fix: http hijacked write error --- pkg/api/router.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/api/router.go b/pkg/api/router.go index 4a3d8594b2c..4aee939774e 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -366,9 +366,7 @@ func (s *Service) mountAPI() { ), }) - handle("/pubsub/{topic}", web.ChainHandlers( - web.FinalHandlerFunc(s.pubsubWsHandler), - )) + handle("/pubsub/{topic}", http.HandlerFunc(s.pubsubWsHandler)) handle("/pubsub/", web.ChainHandlers( web.FinalHandler(jsonhttp.MethodHandler{ From 96983daf1698a6424753a7c51ba3f65d27d52161 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 11:14:03 +0200 Subject: [PATCH 09/51] chore: debugging --- pkg/p2p/libp2p/libp2p.go | 4 ++-- pkg/pubsub/pubsub.go | 5 +++++ pkg/pubsub/ws.go | 14 +++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 9bdc73143cb..fc0cb6abbd3 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -652,7 +652,7 @@ func (s *Service) handleIncoming(stream network.Stream) { if exists := s.peers.addIfNotExists(stream.Conn(), overlay, i.FullNode); exists { s.logger.Debug("stream handler: peer already exists", "peer_address", overlay) if err = handshakeStream.FullClose(); err != nil { - s.logger.Debug("stream handler: could not close stream", "peer_address", overlay, "error", err) + s.logger.Info("stream handler: could not close stream", "peer_address", overlay, "error", err) s.logger.Error(nil, "stream handler: unable to handshake with peer", "peer_address", overlay) _ = stream.Conn().Close() } @@ -660,7 +660,7 @@ func (s *Service) handleIncoming(stream network.Stream) { } if err = handshakeStream.FullClose(); err != nil { - s.logger.Debug("stream handler: could not close stream", "peer_address", overlay, "error", err) + s.logger.Info("stream handler: could not close stream", "peer_address", overlay, "error", err) s.logger.Error(nil, "stream handler: unable to handshake with peer", "peer_address", overlay) _ = s.Disconnect(overlay, "could not fully close stream on handshake") return diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index d56bbbf3c76..a449404709a 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -146,10 +146,12 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr return nil, err } + fmt.Println("Elso1") bzzAddr, err := s.p2p.Connect(ctx, []ma.Multiaddr{underlay}) if err != nil && !errors.Is(err, p2p.ErrAlreadyConnected) { return nil, fmt.Errorf("connect to peer: %w", err) } + fmt.Println("Elso2") stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) if err != nil { @@ -216,6 +218,7 @@ func (s *Service) Topics() []TopicInfo { // brokerHandler handles incoming streams on the broker side. func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.Stream) error { + fmt.Println("Broker handler ") if !s.brokerMode { _ = stream.Reset() return ErrBrokerDisabled @@ -243,6 +246,7 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S } rwBytes := headers[HeaderReadWrite] + fmt.Println("rwBytes header1") if len(rwBytes) != 1 { _ = stream.Reset() return ErrWrongHeaders @@ -250,6 +254,7 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S if rwBytes[0] == 1 { return s.handlePublisher(ctx, peer, stream, bc, headers) } + fmt.Println("rwBytes header2") return s.handleSubscriber(ctx, peer, stream, bc) } diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index ac6ed49d230..cd62320b1fb 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -31,7 +31,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l ) conn.SetCloseHandler(func(code int, text string) error { - logger.Debug("pubsub ws: client gone", "topic", fmt.Sprintf("%x", sc.TopicAddr), "code", code, "message", text) + logger.Info("pubsub ws: client gone", "topic", fmt.Sprintf("%x", sc.TopicAddr), "code", code, "message", text) return nil }) @@ -40,19 +40,19 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l go func() { for { if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil { - logger.Debug("pubsub ws: set read deadline failed", "error", err) + logger.Info("pubsub ws: set read deadline failed", "error", err) break } _, p, err := conn.ReadMessage() if err != nil { if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { - logger.Debug("pubsub ws: read error", "error", err) + logger.Info("pubsub ws: read error", "error", err) } break } if err := writeRaw(sc.Stream, p); err != nil { - logger.Debug("pubsub ws: write to p2p stream failed", "error", err) + logger.Info("pubsub ws: write to p2p stream failed", "error", err) break } } @@ -72,14 +72,14 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l wsPayload, err := sc.Mode.ReadBrokerMessage(sc.Stream) if err != nil { if ctx.Err() == nil { - logger.Debug("pubsub ws: read broker message failed", "error", err) + logger.Info("pubsub ws: read broker message failed", "error", err) } options.Cancel() return } if err := conn.WriteMessage(websocket.BinaryMessage, wsPayload); err != nil { - logger.Debug("pubsub ws: write to ws failed", "error", err) + logger.Info("pubsub ws: write to ws failed", "error", err) options.Cancel() return } @@ -93,7 +93,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l for { if err := conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil { - logger.Debug("pubsub ws: set write deadline failed", "error", err) + logger.Info("pubsub ws: set write deadline failed", "error", err) return } select { From 4f83e7f55cf40652110b007c516e8010ebf4b367 Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Tue, 21 Apr 2026 12:20:18 +0200 Subject: [PATCH 10/51] fix: libp2p backoff clear --- pkg/api/pubsub.go | 6 +++--- pkg/p2p/libp2p/libp2p.go | 33 ++++++++++++++++++++------------- pkg/pubsub/pubsub.go | 4 ++-- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 7a193121684..41796c0bece 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -49,7 +49,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { } underlay, err := ma.NewMultiaddr(peerHeader) if err != nil { - logger.Debug("invalid peer multiaddr", "value", peerHeader, "error", err) + logger.Info("invalid peer multiaddr", "value", peerHeader, "error", err) jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Peer header") return } @@ -100,7 +100,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { subscriberConn, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) if err != nil { cancel() - logger.Debug("pubsub connect failed", "error", err) + logger.Info("pubsub connect failed", "error", err) jsonhttp.InternalServerError(w, "pubsub connect failed") return } @@ -116,7 +116,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { if err != nil { cancel() _ = subscriberConn.Stream.Close() - logger.Debug("websocket upgrade failed", "error", err) + logger.Info("websocket upgrade failed", "error", err) logger.Error(nil, "websocket upgrade failed") jsonhttp.InternalServerError(w, "upgrade failed") return diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index fc0cb6abbd3..462322c0d0e 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -593,7 +593,7 @@ func (s *Service) handleIncoming(stream network.Stream) { peerAddrs, err := s.peerMultiaddrs(s.ctx, peerID) if err != nil { s.logger.Debug("stream handler: handshake: build remote multiaddrs", "peer_id", peerID, "error", err) - s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs", "peer_id", peerID) + s.logger.Error(err, "stream handler: handshake: build remote multiaddrs", "peer_id", peerID) _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) return @@ -608,7 +608,7 @@ func (s *Service) handleIncoming(stream network.Stream) { observedAddrs, err = buildFullMAs([]ma.Multiaddr{stream.Conn().RemoteMultiaddr()}, peerID) if err != nil { s.logger.Debug("stream handler: handshake: build remote multiaddrs fallback", "peer_id", peerID, "error", err) - s.logger.Error(nil, "stream handler: handshake: build remote multiaddrs fallback", "peer_id", peerID) + s.logger.Error(err, "stream handler: handshake: build remote multiaddrs fallback", "peer_id", peerID) _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) return @@ -625,7 +625,7 @@ func (s *Service) handleIncoming(stream network.Stream) { ) if err != nil { s.logger.Debug("stream handler: handshake: handle failed", "peer_id", peerID, "error", err) - s.logger.Error(nil, "stream handler: handshake: handle failed", "peer_id", peerID) + s.logger.Error(err, "stream handler: handshake: handle failed", "peer_id", peerID) _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) return @@ -636,14 +636,14 @@ func (s *Service) handleIncoming(stream network.Stream) { blocked, err := s.blocklist.Exists(overlay) if err != nil { s.logger.Debug("stream handler: blocklisting: exists failed", "peer_address", overlay, "error", err) - s.logger.Error(nil, "stream handler: internal error while connecting with peer", "peer_address", overlay) + s.logger.Error(err, "stream handler: internal error while connecting with peer", "peer_address", overlay) _ = handshakeStream.Reset() _ = stream.Conn().Close() return } if blocked { - s.logger.Error(nil, "stream handler: blocked connection from blocklisted peer", "peer_address", overlay) + s.logger.Error(err, "stream handler: blocked connection from blocklisted peer", "peer_address", overlay) _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(peerID) return @@ -652,16 +652,16 @@ func (s *Service) handleIncoming(stream network.Stream) { if exists := s.peers.addIfNotExists(stream.Conn(), overlay, i.FullNode); exists { s.logger.Debug("stream handler: peer already exists", "peer_address", overlay) if err = handshakeStream.FullClose(); err != nil { - s.logger.Info("stream handler: could not close stream", "peer_address", overlay, "error", err) - s.logger.Error(nil, "stream handler: unable to handshake with peer", "peer_address", overlay) + s.logger.Debug("stream handler: could not close stream", "peer_address", overlay, "error", err) + s.logger.Error(err, "stream handler: unable to handshake with peer", "peer_address", overlay) _ = stream.Conn().Close() } return } if err = handshakeStream.FullClose(); err != nil { - s.logger.Info("stream handler: could not close stream", "peer_address", overlay, "error", err) - s.logger.Error(nil, "stream handler: unable to handshake with peer", "peer_address", overlay) + s.logger.Debug("stream handler: could not close stream", "peer_address", overlay, "error", err) + s.logger.Error(err, "stream handler: unable to handshake with peer", "peer_address", overlay) _ = s.Disconnect(overlay, "could not fully close stream on handshake") return } @@ -671,7 +671,7 @@ func (s *Service) handleIncoming(stream network.Stream) { err = s.addressbook.Put(i.BzzAddress.Overlay, *i.BzzAddress) if err != nil { s.logger.Debug("stream handler: addressbook put error", "peer_id", peerID, "error", err) - s.logger.Error(nil, "stream handler: unable to persist peer", "peer_id", peerID) + s.logger.Error(err, "stream handler: unable to persist peer", "peer_id", peerID) _ = s.Disconnect(i.BzzAddress.Overlay, "unable to persist peer in addressbook") return } @@ -874,7 +874,7 @@ func (s *Service) AddProtocol(p p2p.ProtocolSpec) (err error) { _ = stream.Reset() if err := s.Blocklist(overlay, bpe.Duration(), bpe.Error()); err != nil { logger.Debug("blocklist: could not blocklist peer", "peer_id", peerID, "error", err) - logger.Error(nil, "unable to blocklist peer", "peer_id", peerID) + logger.Error(err, "unable to blocklist peer", "peer_id", peerID) } loggerV1.Debug("handler: peer blocklisted", "protocol", p.Name, "peer_address", overlay) } @@ -1017,6 +1017,13 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b } connectCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + // Clear any stale libp2p swarm dial backoff for this peer so that + // an explicit Connect call always attempts a real TCP dial rather + // than failing immediately with a cached backoff error (which would + // still count against the Bee connection breaker). + if sw, ok := s.host.Network().(*lp2pswarm.Swarm); ok { + sw.Backoff().Clear(info.ID) + } err = s.connectionBreaker.Execute(func() error { return s.host.Connect(connectCtx, *info) }) cancel() @@ -1103,14 +1110,14 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b blocked, err := s.blocklist.Exists(overlay) if err != nil { s.logger.Debug("blocklisting: exists failed", "peer_id", info.ID, "error", err) - s.logger.Error(nil, "internal error while connecting with peer", "peer_id", info.ID) + s.logger.Error(err, "internal error while connecting with peer", "peer_id", info.ID) _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(info.ID) return nil, err } if blocked { - s.logger.Error(nil, "blocked connection to blocklisted peer", "peer_id", info.ID) + s.logger.Error(err, "blocked connection to blocklisted peer", "peer_id", info.ID) _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(info.ID) return nil, p2p.ErrPeerBlocklisted diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index a449404709a..9d44d126738 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -146,12 +146,12 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr return nil, err } - fmt.Println("Elso1") + s.logger.Info("connecting to broker peer", "underlay", underlay) bzzAddr, err := s.p2p.Connect(ctx, []ma.Multiaddr{underlay}) if err != nil && !errors.Is(err, p2p.ErrAlreadyConnected) { return nil, fmt.Errorf("connect to peer: %w", err) } - fmt.Println("Elso2") + s.logger.Info("connected to broker peer", "overlay", bzzAddr.Overlay) stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) if err != nil { From cba55c8437f9d40e60426b54fbb7edb7a0be86e7 Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Tue, 21 Apr 2026 12:37:53 +0200 Subject: [PATCH 11/51] fix: http hijack handler --- pkg/log/httpaccess/http_access.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/log/httpaccess/http_access.go b/pkg/log/httpaccess/http_access.go index fbc350dba60..19663b8118d 100644 --- a/pkg/log/httpaccess/http_access.go +++ b/pkg/log/httpaccess/http_access.go @@ -118,9 +118,16 @@ func (rr *responseRecorder) CloseNotify() <-chan bool { return rr.ResponseWriter.(http.CloseNotifier).CloseNotify() } -// Hijack implements http.Hijacker. +// Hijack implements http.Hijacker so that WebSocket upgrades pass through +// correctly. Without this, the underlying connection is hijacked by the +// upgrader but the recorder still holds a reference to the (now-hijacked) +// ResponseWriter, causing "response.Write on hijacked connection" errors. func (rr *responseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { - return rr.ResponseWriter.(http.Hijacker).Hijack() + h, ok := rr.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return h.Hijack() } // Flush implements http.Flusher. From 4c08e7235d162d0a999b7879b48e27897ad7fe97 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 14:11:36 +0200 Subject: [PATCH 12/51] fix: hijacked websocket error --- pkg/log/httpaccess/http_access.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pkg/log/httpaccess/http_access.go b/pkg/log/httpaccess/http_access.go index 19663b8118d..b30803e78b3 100644 --- a/pkg/log/httpaccess/http_access.go +++ b/pkg/log/httpaccess/http_access.go @@ -92,12 +92,16 @@ type responseRecorder struct { http.ResponseWriter // Metrics. - status int - size int + status int + size int + hijacked bool } // Write implements http.ResponseWriter. func (rr *responseRecorder) Write(b []byte) (int, error) { + if rr.hijacked { + return 0, http.ErrHijacked + } size, err := rr.ResponseWriter.Write(b) rr.size += size return size, err @@ -105,6 +109,9 @@ func (rr *responseRecorder) Write(b []byte) (int, error) { // WriteHeader implements http.ResponseWriter. func (rr *responseRecorder) WriteHeader(s int) { + if rr.hijacked { + return + } rr.ResponseWriter.WriteHeader(s) if rr.status == 0 { rr.status = s @@ -127,11 +134,18 @@ func (rr *responseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { if !ok { return nil, nil, http.ErrNotSupported } - return h.Hijack() + conn, brw, err := h.Hijack() + if err == nil { + rr.hijacked = true + } + return conn, brw, err } // Flush implements http.Flusher. func (rr *responseRecorder) Flush() { + if rr.hijacked { + return + } rr.ResponseWriter.(http.Flusher).Flush() } From 54a62a8332a5633cd10d41f044b560af3373e56b Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 14:12:15 +0200 Subject: [PATCH 13/51] chore: logs debugging --- pkg/api/pubsub.go | 5 ++++- pkg/pubsub/pubsub.go | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 41796c0bece..0554b3cc2be 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -112,15 +112,18 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { CheckOrigin: s.checkOrigin, } + logger.Info("upgrading to websocket") conn, err := upgrader.Upgrade(w, r, nil) if err != nil { cancel() _ = subscriberConn.Stream.Close() logger.Info("websocket upgrade failed", "error", err) logger.Error(nil, "websocket upgrade failed") - jsonhttp.InternalServerError(w, "upgrade failed") + // NOTE: Upgrade() hijacks the connection before returning an error, + // so do NOT write an HTTP response here. return } + logger.Info("websocket upgrade successful") pingPeriod := headers.KeepAlive * time.Second if pingPeriod == 0 { diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 9d44d126738..c0d9d41a016 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -344,11 +344,13 @@ func (s *Service) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p rawMsg, err := bc.mode.ReadPublisherMessage(stream) if err != nil { if errors.Is(err, io.EOF) { + s.logger.Info("publisher stream EOF", "peer", peer.Address) return nil } return fmt.Errorf("read publisher message: %w", err) } + s.logger.Info("publisher message received", "peer", peer.Address, "size", len(rawMsg)) s.broadcastToSubscribers(bc, rawMsg) } } @@ -358,11 +360,13 @@ func (s *Service) broadcastToSubscribers(bc *brokerConn, rawMsg []byte) { bc.mu.Lock() defer bc.mu.Unlock() + s.logger.Info("broadcasting to subscribers", "count", len(bc.subscribers), "size", len(rawMsg)) for _, sub := range bc.subscribers { msg := bc.mode.FormatBroadcast(bc, sub, rawMsg) select { case sub.outCh <- msg: + s.logger.Info("message enqueued for subscriber", "peer", sub.overlay, "size", len(msg)) default: s.logger.Warning("subscriber message queue full, dropping message", "peer", sub.overlay) } From 92cf330c8a55e8bd7e067a84592dd623c9a8a09c Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 14:12:34 +0200 Subject: [PATCH 14/51] fix: pingpong with subscriber --- pkg/pubsub/ws.go | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index cd62320b1fb..9ec52d5faa6 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -30,41 +30,48 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l readDeadline = options.PingPeriod + time.Second ) + logger.Info("pubsub ws: starting", "topic", fmt.Sprintf("%x", sc.TopicAddr), "isPublisher", isPublisher, "pingPeriod", options.PingPeriod) + conn.SetCloseHandler(func(code int, text string) error { logger.Info("pubsub ws: client gone", "topic", fmt.Sprintf("%x", sc.TopicAddr), "code", code, "message", text) return nil }) - // If Publisher, read from WebSocket and write to p2p stream (send to Broker). - if isPublisher { - go func() { - for { - if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil { - logger.Info("pubsub ws: set read deadline failed", "error", err) - break - } - _, p, err := conn.ReadMessage() - if err != nil { - if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { - logger.Info("pubsub ws: read error", "error", err) - } - break + // A read loop is always required so gorilla can process pong responses + // and close frames from the client. + go func() { + for { + if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil { + logger.Info("pubsub ws: set read deadline failed", "error", err) + break + } + msgType, p, err := conn.ReadMessage() + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) { + logger.Info("pubsub ws: read error", "error", err) + } else { + logger.Info("pubsub ws: read loop ended", "error", err) } + break + } + if isPublisher { + logger.Info("pubsub ws: publisher message from ws", "type", msgType, "size", len(p)) if err := writeRaw(sc.Stream, p); err != nil { logger.Info("pubsub ws: write to p2p stream failed", "error", err) break } } - options.Cancel() - }() - } + } + options.Cancel() + }() // Read from p2p stream (Broker messages) and forward to WebSocket. go func() { for { select { case <-ctx.Done(): + logger.Info("pubsub ws: p2p reader context done") return default: } @@ -78,6 +85,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l return } + logger.Info("pubsub ws: forwarding broker message to ws", "size", len(wsPayload)) if err := conn.WriteMessage(websocket.BinaryMessage, wsPayload); err != nil { logger.Info("pubsub ws: write to ws failed", "error", err) options.Cancel() @@ -89,6 +97,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l defer func() { ticker.Stop() _ = conn.Close() + logger.Info("pubsub ws: closed", "topic", fmt.Sprintf("%x", sc.TopicAddr)) }() for { @@ -98,10 +107,12 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l } select { case <-ctx.Done(): + logger.Info("pubsub ws: context cancelled, closing") _ = conn.WriteMessage(websocket.CloseMessage, []byte{}) return case <-ticker.C: if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil { + logger.Info("pubsub ws: ping failed, closing", "error", err) return } } From 24accc91db4191c1521f1d110d1c7111f188d1df Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Tue, 21 Apr 2026 14:14:49 +0200 Subject: [PATCH 15/51] fix: upgrade ws no compress --- pkg/api/router.go | 20 +++++++++++++++++++- pkg/pubsub/pubsub.go | 1 + pkg/pubsub/ws.go | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/api/router.go b/pkg/api/router.go index 4aee939774e..b897ccb0fe6 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -44,7 +44,15 @@ func (s *Service) Mount() { s.Handler = web.ChainHandlers( httpaccess.NewHTTPAccessLogHandler(s.logger, s.tracer, "api access"), - handlers.CompressHandler, + func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + h.ServeHTTP(w, r) + return + } + handlers.CompressHandler(h).ServeHTTP(w, r) + }) + }, s.corsHandler, web.NoCacheHeadersHandler, web.FinalHandler(router), @@ -74,6 +82,16 @@ func (s *Service) EnableFullAPI() { } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Skip compression for WebSocket upgrade requests. + // CompressHandler wraps the ResponseWriter with a gzip writer; when + // the WebSocket upgrader hijacks the connection and the handler returns, + // the gzip writer tries to flush/close and writes to the hijacked + // connection, causing "response.Write on hijacked connection" errors. + if strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + h.ServeHTTP(w, r) + return + } + // Skip compression for GET requests on download endpoints. // This is done in order to preserve Content-Length header in response, // because CompressHandler is always removing it. diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index c0d9d41a016..c8a0849ec96 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -155,6 +155,7 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) if err != nil { + s.logger.Error(err,"bagoy open stream") return nil, fmt.Errorf("open stream: %w", err) } diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index 9ec52d5faa6..14b31244915 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -34,6 +34,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l conn.SetCloseHandler(func(code int, text string) error { logger.Info("pubsub ws: client gone", "topic", fmt.Sprintf("%x", sc.TopicAddr), "code", code, "message", text) + options.Cancel() return nil }) From a30f1dc787ecfe44ffe99a4b1861234b0d2b65c6 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 15:05:39 +0200 Subject: [PATCH 16/51] fix: readdeadline on pong --- pkg/pubsub/ws.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index 14b31244915..5bd93cfb6a3 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -38,6 +38,14 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l return nil }) + // Reset read deadline on every pong so idle subscribers don't time out. + conn.SetPongHandler(func(appData string) error { + if err := conn.SetReadDeadline(time.Now().Add(readDeadline)); err != nil { + return err + } + return conn.SetWriteDeadline(time.Now().Add(writeDeadline)) + }) + // A read loop is always required so gorilla can process pong responses // and close frames from the client. go func() { From b59e6e2884c7b72eeee6fcd60897a00a8e7ed02d Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 15:07:44 +0200 Subject: [PATCH 17/51] fix(broker): eof return --- pkg/pubsub/pubsub.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index c8a0849ec96..dcd5f985166 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -155,7 +155,7 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) if err != nil { - s.logger.Error(err,"bagoy open stream") + s.logger.Error(err, "bagoy open stream") return nil, fmt.Errorf("open stream: %w", err) } @@ -346,9 +346,9 @@ func (s *Service) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p if err != nil { if errors.Is(err, io.EOF) { s.logger.Info("publisher stream EOF", "peer", peer.Address) - return nil + } else { + return fmt.Errorf("read publisher message: %w", err) } - return fmt.Errorf("read publisher message: %w", err) } s.logger.Info("publisher message received", "peer", peer.Address, "size", len(rawMsg)) From b108e6959c86fe06e0ea81a439d4938f0e6859e8 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 21 Apr 2026 15:12:50 +0200 Subject: [PATCH 18/51] chore: logs --- pkg/pubsub/pubsub.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index dcd5f985166..901e7dc912a 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -219,7 +219,7 @@ func (s *Service) Topics() []TopicInfo { // brokerHandler handles incoming streams on the broker side. func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.Stream) error { - fmt.Println("Broker handler ") + s.logger.Debug("broker handler invoked", "peer", peer.Address) if !s.brokerMode { _ = stream.Reset() return ErrBrokerDisabled @@ -247,7 +247,7 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S } rwBytes := headers[HeaderReadWrite] - fmt.Println("rwBytes header1") + s.logger.Debug("reading rw header", "peer", peer.Address) if len(rwBytes) != 1 { _ = stream.Reset() return ErrWrongHeaders @@ -255,7 +255,7 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S if rwBytes[0] == 1 { return s.handlePublisher(ctx, peer, stream, bc, headers) } - fmt.Println("rwBytes header2") + s.logger.Debug("handling as subscriber", "peer", peer.Address) return s.handleSubscriber(ctx, peer, stream, bc) } From 7a3125245f614d355e76c81a0971f767bc658bc1 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 12:02:41 +0200 Subject: [PATCH 19/51] fix: wrong connection mapping and many refactors --- pkg/api/pubsub.go | 13 ++- pkg/pubsub/mode.go | 231 ++++++++++++++++++++++++++++++++++---- pkg/pubsub/pubsub.go | 261 +++++++++---------------------------------- pkg/pubsub/ws.go | 14 +-- 4 files changed, 277 insertions(+), 242 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 0554b3cc2be..a6821ea24a9 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -97,14 +97,13 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { // Connect to broker peer ctx, cancel := context.WithCancel(context.Background()) - subscriberConn, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) + mode, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) if err != nil { cancel() logger.Info("pubsub connect failed", "error", err) jsonhttp.InternalServerError(w, "pubsub connect failed") return } - // Upgrade to WebSocket upgrader := websocket.Upgrader{ ReadBufferSize: swarm.ChunkWithSpanSize, @@ -116,7 +115,9 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { cancel() - _ = subscriberConn.Stream.Close() + if sc := mode.GetSubscriberConn(); sc != nil { + _ = sc.Stream.Close() + } logger.Info("websocket upgrade failed", "error", err) logger.Error(nil, "websocket upgrade failed") // NOTE: Upgrade() hijacks the connection before returning an error, @@ -134,9 +135,11 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { s.wsWg.Add(1) go func() { - pubsub.ListeningWs(ctx, conn, pubsub.WsOptions{PingPeriod: pingPeriod, Cancel: cancel}, logger, subscriberConn, isPublisher) + pubsub.ListeningWs(ctx, conn, pubsub.WsOptions{PingPeriod: pingPeriod, Cancel: cancel}, logger, mode, isPublisher) _ = conn.Close() - subscriberConn.Cancel() + if sc := mode.GetSubscriberConn(); sc != nil { + sc.Cancel() + } s.wsWg.Done() }() } diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 0d74777c470..07f1d99b3fe 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -11,7 +11,9 @@ import ( "errors" "fmt" "io" + "sync" + "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -33,17 +35,18 @@ type ModeID uint8 type Mode interface { ID() ModeID TopicAddress() swarm.Address - Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) - - // Broker side - publisher - ValidatePublisher(bc *brokerConn, headers p2p.Headers) error - ReadPublisherMessage(stream p2p.Stream) ([]byte, error) - // Broker side - broadcast - FormatBroadcast(bt *brokerConn, sub *brokerSubscriber, rawMsg []byte) []byte - - // Subscriber/WS side + // Subscriber side - outbound connection to broker + Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) + CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) + GetSubscriberConn() *SubscriberConn + RemoveSubscriberConn() ReadBrokerMessage(stream p2p.Stream) ([]byte, error) + + // Broker side - handles incoming streams (publisher and subscriber) + HandleBroker(ctx context.Context, peer p2p.Peer, stream p2p.Stream, headers p2p.Headers) error + SubscriberCount() int + SubscriberOverlays() []string } // --- GSOC Ephemeral Mode (mode 1) --- @@ -56,17 +59,38 @@ const ( // GSOCEphemeralMode implements Mode for GSOC ephemeral messaging. type GSOCEphemeralMode struct { - topicAddress swarm.Address - gsocOwner []byte - gsocID []byte - handshakeHappened bool // true after the first message from broker. sends validation metadata + mu sync.RWMutex + topicAddress swarm.Address + gsocOwner []byte + gsocID []byte + logger log.Logger + subscribers map[string]*brokerSubscriber + subscriberConn *SubscriberConn +} + +// brokerSubscriber holds a subscriber's stream and outgoing message channel. +type brokerSubscriber struct { + overlay swarm.Address + stream p2p.Stream + outCh chan []byte + cancel context.CancelFunc + handshakeHappened bool +} + +// SubscriberConn represents the subscriber-side connection to a broker. +type SubscriberConn struct { + Stream p2p.Stream + Overlay swarm.Address + Cancel context.CancelFunc } var _ Mode = (*GSOCEphemeralMode)(nil) -func NewGSOCEphemeralMode(topicAddress []byte) *GSOCEphemeralMode { +func NewGSOCEphemeralMode(topicAddress []byte, logger log.Logger) *GSOCEphemeralMode { return &GSOCEphemeralMode{ topicAddress: swarm.NewAddress(topicAddress), + logger: logger, + subscribers: make(map[string]*brokerSubscriber), } } @@ -93,15 +117,15 @@ func (m *GSOCEphemeralMode) Connect(ctx context.Context, p p2p.Streamer, overlay return p.NewStream(ctx, overlay, headers, protocolName, protocolVersion, streamName) } -// ValidatePublisher sets SOC parameters on the broker side so it can validate the messages. -func (m *GSOCEphemeralMode) ValidatePublisher(bc *brokerConn, headers p2p.Headers) error { +// validatePublisher sets SOC parameters on the broker side so it can validate the messages. +func (m *GSOCEphemeralMode) validatePublisher(headers p2p.Headers) error { gsocOwner := headers[HeaderGsocOwner] gsocID := headers[HeaderGsocID] - bc.mu.Lock() + m.mu.Lock() m.setGsocParams(gsocOwner, gsocID) set := m.gsocID != nil - bc.mu.Unlock() + m.mu.Unlock() if !set { return ErrWrongHeaders @@ -111,15 +135,15 @@ func (m *GSOCEphemeralMode) ValidatePublisher(bc *brokerConn, headers p2p.Header // FormatBroadcast formats a raw publisher message for delivery to a subscriber. // First delivery to each subscriber includes a handshake with SOC identity; subsequent are data-only. -func (m *GSOCEphemeralMode) FormatBroadcast(bt *brokerConn, sub *brokerSubscriber, rawMsg []byte) []byte { - if !m.handshakeHappened { +func (m *GSOCEphemeralMode) formatBroadcast(sub *brokerSubscriber, rawMsg []byte) []byte { + if !sub.handshakeHappened { // Handshake: [1B type=0x01][32B SOC ID][20B owner][65B sig][4B span][NB payload] msg := make([]byte, 1+IDSize+OwnerSize+len(rawMsg)) msg[0] = MsgTypeHandshake copy(msg[1:1+IDSize], m.gsocID) copy(msg[1+IDSize:1+IDSize+OwnerSize], m.gsocOwner) copy(msg[1+IDSize+OwnerSize:], rawMsg) - m.handshakeHappened = true + sub.handshakeHappened = true return msg } @@ -195,6 +219,171 @@ func (m *GSOCEphemeralMode) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) } } +// SubscriberCount returns the number of active subscribers. +func (m *GSOCEphemeralMode) SubscriberCount() int { + m.mu.RLock() + defer m.mu.RUnlock() + return len(m.subscribers) +} + +// SubscriberOverlays returns the overlay addresses of all active subscribers. +func (m *GSOCEphemeralMode) SubscriberOverlays() []string { + m.mu.RLock() + defer m.mu.RUnlock() + overlays := make([]string, 0, len(m.subscribers)) + for _, sub := range m.subscribers { + overlays = append(overlays, sub.overlay.String()) + } + return overlays +} + +// broadcast sends a message to all subscribers. +func (m *GSOCEphemeralMode) broadcast(rawMsg []byte) { + m.mu.Lock() + defer m.mu.Unlock() + + m.logger.Info("broadcasting to subscribers", "count", len(m.subscribers), "size", len(rawMsg)) + for _, sub := range m.subscribers { + msg := m.formatBroadcast(sub, rawMsg) + + select { + case sub.outCh <- msg: + m.logger.Info("message enqueued for subscriber", "peer", sub.overlay, "size", len(msg)) + default: + m.logger.Warning("subscriber message queue full, dropping message", "peer", sub.overlay) + } + } +} + +// HandleBroker handles an incoming broker-side stream, dispatching to publisher or subscriber handling. +func (m *GSOCEphemeralMode) HandleBroker(ctx context.Context, peer p2p.Peer, stream p2p.Stream, headers p2p.Headers) error { + rwBytes := headers[HeaderReadWrite] + m.logger.Debug("reading rw header", "peer", peer.Address) + if len(rwBytes) != 1 { + _ = stream.Reset() + return ErrWrongHeaders + } + if rwBytes[0] == 1 { + return m.handlePublisher(ctx, peer, stream, headers) + } + m.logger.Debug("handling as subscriber", "peer", peer.Address) + return m.handleSubscriber(ctx, peer, stream) +} + +func (m *GSOCEphemeralMode) handleSubscriber(ctx context.Context, peer p2p.Peer, stream p2p.Stream) error { + subCtx, cancel, unregister := m.registerSubscriber(ctx, peer.Address, stream) + defer cancel() + defer unregister() + + m.logger.Debug("subscriber connected", "peer", peer.Address, "topic", m.TopicAddress()) + + <-subCtx.Done() + if errors.Is(subCtx.Err(), context.Canceled) { + return nil + } + return subCtx.Err() +} + +func (m *GSOCEphemeralMode) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p.Stream, headers p2p.Headers) error { + if err := m.validatePublisher(headers); err != nil { + _ = stream.Reset() + return err + } + + partCtx, cancel, unregister := m.registerSubscriber(ctx, peer.Address, stream) + defer cancel() + defer unregister() + + m.logger.Debug("publisher connected", "peer", peer.Address, "topic", m.TopicAddress()) + + for { + select { + case <-partCtx.Done(): + if errors.Is(partCtx.Err(), context.Canceled) { + return nil + } + return partCtx.Err() + default: + } + + rawMsg, err := m.ReadPublisherMessage(stream) + if err != nil { + if errors.Is(err, io.EOF) { + m.logger.Info("publisher stream EOF", "peer", peer.Address) + } else { + return fmt.Errorf("read publisher message: %w", err) + } + } + + m.logger.Info("publisher message received", "peer", peer.Address, "size", len(rawMsg)) + m.broadcast(rawMsg) + } +} + +// registerSubscriber adds a peer as a subscriber and starts a write goroutine for it. +func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swarm.Address, stream p2p.Stream) (context.Context, context.CancelFunc, func()) { + connCtx, cancel := context.WithCancel(ctx) + + sub := &brokerSubscriber{ + overlay: overlay, + stream: stream, + outCh: make(chan []byte, 256), + cancel: cancel, + } + + overlayKey := overlay.String() + m.mu.Lock() + m.subscribers[overlayKey] = sub + m.mu.Unlock() + + go func() { + for { + select { + case <-connCtx.Done(): + return + case msg := <-sub.outCh: + if err := writeRaw(stream, msg); err != nil { + cancel() + return + } + } + } + }() + + unregister := func() { + m.mu.Lock() + delete(m.subscribers, overlayKey) + m.mu.Unlock() + } + + return connCtx, cancel, unregister +} + +// CreateSubscriberConn creates and stores the subscriber-side connection in this mode. +func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) { + m.mu.Lock() + m.subscriberConn = &SubscriberConn{ + Stream: stream, + Overlay: overlay, + Cancel: cancel, + } + m.mu.Unlock() +} + +// GetSubscriberConn returns the subscriber-side connection, or nil. +func (m *GSOCEphemeralMode) GetSubscriberConn() *SubscriberConn { + m.mu.RLock() + defer m.mu.RUnlock() + return m.subscriberConn +} + +// RemoveSubscriberConn clears the subscriber-side connection. +func (m *GSOCEphemeralMode) RemoveSubscriberConn() { + m.mu.Lock() + m.subscriberConn = nil + m.mu.Unlock() +} + // setGsocParams sets the GSOC recurring parameters so that messages don't need to include them. func (m *GSOCEphemeralMode) setGsocParams(gsocOwner, gsocID []byte) { if m.gsocOwner != nil { diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 901e7dc912a..d0583b06c40 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "io" "sync" "github.com/ethersphere/bee/v2/pkg/crypto" @@ -48,10 +47,10 @@ var ( ErrTopicMismatch = errors.New("pubsub: topic address mismatch") ) -func newMode(topicAddr [32]byte, modeID ModeID) (Mode, error) { +func newMode(topicAddr [32]byte, modeID ModeID, logger log.Logger) (Mode, error) { switch modeID { case ModeGSOCEphemeral: - return NewGSOCEphemeralMode(topicAddr[:]), nil + return NewGSOCEphemeralMode(topicAddr[:], logger), nil default: return nil, fmt.Errorf("pubsub: unknown mode: %d", modeID) } @@ -72,28 +71,10 @@ type TopicInfo struct { Connections []string `json:"connections"` } -// brokerSubscriber holds a subscriber's stream and outgoing message channel. -type brokerSubscriber struct { - overlay swarm.Address - stream p2p.Stream - outCh chan []byte - cancel context.CancelFunc -} - -// brokerConn manages all connections for a single topic on the broker side. -type brokerConn struct { - mu sync.RWMutex - mode Mode - subscribers map[string]*brokerSubscriber -} - -// SubscriberConn represents the subscriber-side connection to a broker. -type SubscriberConn struct { - Stream p2p.Stream +// TopicModeKey is a composite key for identifying a mode instance per topic. +type TopicModeKey struct { TopicAddr [32]byte - Mode Mode - Overlay swarm.Address - Cancel context.CancelFunc + ModeID ModeID } // P2P groups the p2p capabilities needed by the pubsub service. @@ -104,23 +85,21 @@ type P2P interface { // Service is the pubsub protocol service. type Service struct { - mu sync.RWMutex - p2p P2P - logger log.Logger - brokerMode bool - maxConns int - brokerConns map[[32]byte]*brokerConn // topic address -> broker connection - subscriberConns map[[32]byte]*SubscriberConn // topic address -> subscriber connection + mu sync.RWMutex + p2p P2P + logger log.Logger + brokerMode bool + maxConns int + modes map[TopicModeKey]Mode // (topic, mode) -> mode instance } func New(p2p P2P, logger log.Logger, brokerMode bool, maxConns int) *Service { s := &Service{ - p2p: p2p, - logger: logger.WithName(loggerName).Register(), - brokerMode: brokerMode, - maxConns: maxConns, - brokerConns: make(map[[32]byte]*brokerConn), - subscriberConns: make(map[[32]byte]*SubscriberConn), + p2p: p2p, + logger: logger.WithName(loggerName).Register(), + brokerMode: brokerMode, + maxConns: maxConns, + modes: make(map[TopicModeKey]Mode), } return s } @@ -140,8 +119,9 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } // Connect establishes a subscriber connection to a broker peer. -func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr [32]byte, modeID ModeID, opts ConnectOptions) (*SubscriberConn, error) { - m, err := newMode(topicAddr, modeID) +func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr [32]byte, modeID ModeID, opts ConnectOptions) (Mode, error) { + key := TopicModeKey{TopicAddr: topicAddr, ModeID: modeID} + m, err := s.getOrCreateMode(key) if err != nil { return nil, err } @@ -155,32 +135,20 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) if err != nil { - s.logger.Error(err, "bagoy open stream") + s.logger.Error(err, "open stream failed") return nil, fmt.Errorf("open stream: %w", err) } connCtx, cancel := context.WithCancel(ctx) - sc := &SubscriberConn{ - Stream: stream, - TopicAddr: topicAddr, - Mode: m, - Overlay: bzzAddr.Overlay, - Cancel: cancel, - } - - s.mu.Lock() - s.subscriberConns[topicAddr] = sc - s.mu.Unlock() + m.CreateSubscriberConn(stream, bzzAddr.Overlay, cancel) go func() { <-connCtx.Done() - s.mu.Lock() - delete(s.subscriberConns, topicAddr) - s.mu.Unlock() _ = stream.FullClose() + m.RemoveSubscriberConn() }() - return sc, nil + return m, nil } // Topics returns info about all active topics. @@ -190,28 +158,26 @@ func (s *Service) Topics() []TopicInfo { var topics []TopicInfo - for addr, bt := range s.brokerConns { - bt.mu.RLock() - conns := make([]string, 0, len(bt.subscribers)) - for _, sub := range bt.subscribers { - conns = append(conns, sub.overlay.String()) + for key, m := range s.modes { + info := TopicInfo{ + TopicAddress: fmt.Sprintf("%x", key.TopicAddr), + Mode: m.ID(), + Connections: m.SubscriberOverlays(), } - bt.mu.RUnlock() - topics = append(topics, TopicInfo{ - TopicAddress: fmt.Sprintf("%x", addr), - Mode: bt.mode.ID(), - Role: "broker", - Connections: conns, - }) - } - - for addr, sc := range s.subscriberConns { - topics = append(topics, TopicInfo{ - TopicAddress: fmt.Sprintf("%x", addr), - Mode: sc.Mode.ID(), - Role: "subscriber", - Connections: []string{sc.Overlay.String()}, - }) + sc := m.GetSubscriberConn() + switch { + case m.SubscriberCount() > 0 && sc != nil: + info.Role = "broker+subscriber" + info.Connections = append(info.Connections, sc.Overlay.String()) + case m.SubscriberCount() > 0: + info.Role = "broker" + case sc != nil: + info.Role = "subscriber" + info.Connections = []string{sc.Overlay.String()} + default: + continue + } + topics = append(topics, info) } return topics @@ -240,159 +206,36 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S _ = stream.Reset() return ErrWrongHeaders } - bc, err := s.getOrCreateBrokerConn(topicAddr, ModeID(modeBytes[0])) + key := TopicModeKey{TopicAddr: topicAddr, ModeID: ModeID(modeBytes[0])} + m, err := s.getOrCreateMode(key) if err != nil { _ = stream.Reset() return err } - rwBytes := headers[HeaderReadWrite] - s.logger.Debug("reading rw header", "peer", peer.Address) - if len(rwBytes) != 1 { - _ = stream.Reset() - return ErrWrongHeaders - } - if rwBytes[0] == 1 { - return s.handlePublisher(ctx, peer, stream, bc, headers) - } - s.logger.Debug("handling as subscriber", "peer", peer.Address) - return s.handleSubscriber(ctx, peer, stream, bc) -} - -// registerBrokerConn registers the peer as a broadcast recipient on bc, starts a write goroutine, -// and returns the connection context, its cancel func, and an unregister func to call on exit. -func (s *Service) registerBrokerConn(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn) (connCtx context.Context, cancel context.CancelFunc, unregister func()) { - connCtx, cancel = context.WithCancel(ctx) - - conn := &brokerSubscriber{ - overlay: peer.Address, - stream: stream, - outCh: make(chan []byte, 256), - cancel: cancel, - } - - overlayKey := peer.Address.String() - bc.mu.Lock() - bc.subscribers[overlayKey] = conn - bc.mu.Unlock() - - go func() { - for { - select { - case <-connCtx.Done(): - return - case msg := <-conn.outCh: - if err := writeRaw(stream, msg); err != nil { - cancel() - return - } - } - } - }() - - unregister = func() { - bc.mu.Lock() - delete(bc.subscribers, overlayKey) - bc.mu.Unlock() - } - return -} - -func (s *Service) handleSubscriber(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn) error { - bc.mu.RLock() - connCount := len(bc.subscribers) - bc.mu.RUnlock() - if s.maxConns > 0 && connCount >= s.maxConns { + if s.maxConns > 0 && m.SubscriberCount() >= s.maxConns { _ = stream.Reset() return ErrMaxConnections } - subCtx, cancel, unregister := s.registerBrokerConn(ctx, peer, stream, bc) - defer cancel() - defer unregister() - - s.logger.Debug("subscriber connected", "peer", peer.Address, "topic", bc.mode.TopicAddress()) - - <-subCtx.Done() - if errors.Is(subCtx.Err(), context.Canceled) { - return nil - } - return subCtx.Err() + return m.HandleBroker(ctx, peer, stream, headers) } -func (s *Service) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p.Stream, bc *brokerConn, headers p2p.Headers) error { - if err := bc.mode.ValidatePublisher(bc, headers); err != nil { - _ = stream.Reset() - return err - } - - partCtx, cancel, unregister := s.registerBrokerConn(ctx, peer, stream, bc) - defer cancel() - defer unregister() - - s.logger.Debug("publisher connected", "peer", peer.Address, "topic", bc.mode.TopicAddress()) - - for { - select { - case <-partCtx.Done(): - if errors.Is(partCtx.Err(), context.Canceled) { - return nil - } - return partCtx.Err() - default: - } - - rawMsg, err := bc.mode.ReadPublisherMessage(stream) - if err != nil { - if errors.Is(err, io.EOF) { - s.logger.Info("publisher stream EOF", "peer", peer.Address) - } else { - return fmt.Errorf("read publisher message: %w", err) - } - } - - s.logger.Info("publisher message received", "peer", peer.Address, "size", len(rawMsg)) - s.broadcastToSubscribers(bc, rawMsg) - } -} - -// broadcastToSubscribers sends a message to all subscribers of a topic. -func (s *Service) broadcastToSubscribers(bc *brokerConn, rawMsg []byte) { - bc.mu.Lock() - defer bc.mu.Unlock() - - s.logger.Info("broadcasting to subscribers", "count", len(bc.subscribers), "size", len(rawMsg)) - for _, sub := range bc.subscribers { - msg := bc.mode.FormatBroadcast(bc, sub, rawMsg) - - select { - case sub.outCh <- msg: - s.logger.Info("message enqueued for subscriber", "peer", sub.overlay, "size", len(msg)) - default: - s.logger.Warning("subscriber message queue full, dropping message", "peer", sub.overlay) - } - } -} - -func (s *Service) getOrCreateBrokerConn(topicAddr [32]byte, modeID ModeID) (*brokerConn, error) { +func (s *Service) getOrCreateMode(key TopicModeKey) (Mode, error) { s.mu.Lock() defer s.mu.Unlock() - if bt, ok := s.brokerConns[topicAddr]; ok { - return bt, nil + if m, ok := s.modes[key]; ok { + return m, nil } - m, err := newMode(topicAddr, modeID) + m, err := newMode(key.TopicAddr, key.ModeID, s.logger) if err != nil { return nil, err } - bc := &brokerConn{ - mode: m, - subscribers: make(map[string]*brokerSubscriber), - } - s.brokerConns[topicAddr] = bc - return bc, nil + s.modes[key] = m + return m, nil } // writeRaw writes raw bytes to the stream. diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index 5bd93cfb6a3..78f9fdbfdeb 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -6,7 +6,6 @@ package pubsub import ( "context" - "fmt" "time" "github.com/ethersphere/bee/v2/pkg/log" @@ -19,21 +18,22 @@ type WsOptions struct { } // ListeningWs bridges a subscriber's p2p stream to a WebSocket connection. -// The Mode on sc.Mode handles all wire-format details: reading broker messages, +// The Mode handles all wire-format details: reading broker messages, // verifying them, and returning the payload to forward to the WebSocket. // If the subscriber is a Publisher, it also reads from the WebSocket // and writes raw messages to the p2p stream. -func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, logger log.Logger, sc *SubscriberConn, isPublisher bool) { +func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, logger log.Logger, mode Mode, isPublisher bool) { + sc := mode.GetSubscriberConn() var ( ticker = time.NewTicker(options.PingPeriod) writeDeadline = options.PingPeriod + time.Second readDeadline = options.PingPeriod + time.Second ) - logger.Info("pubsub ws: starting", "topic", fmt.Sprintf("%x", sc.TopicAddr), "isPublisher", isPublisher, "pingPeriod", options.PingPeriod) + logger.Info("pubsub ws: starting", "topic", mode.TopicAddress(), "isPublisher", isPublisher, "pingPeriod", options.PingPeriod) conn.SetCloseHandler(func(code int, text string) error { - logger.Info("pubsub ws: client gone", "topic", fmt.Sprintf("%x", sc.TopicAddr), "code", code, "message", text) + logger.Info("pubsub ws: client gone", "topic", mode.TopicAddress(), "code", code, "message", text) options.Cancel() return nil }) @@ -85,7 +85,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l default: } - wsPayload, err := sc.Mode.ReadBrokerMessage(sc.Stream) + wsPayload, err := mode.ReadBrokerMessage(sc.Stream) if err != nil { if ctx.Err() == nil { logger.Info("pubsub ws: read broker message failed", "error", err) @@ -106,7 +106,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l defer func() { ticker.Stop() _ = conn.Close() - logger.Info("pubsub ws: closed", "topic", fmt.Sprintf("%x", sc.TopicAddr)) + logger.Info("pubsub ws: closed", "topic", mode.TopicAddress()) }() for { From 525f6d87c57eade6d11a64d78e9ea63719467dee Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 12:38:29 +0200 Subject: [PATCH 20/51] fix: eof in websocket --- pkg/pubsub/mode.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 07f1d99b3fe..3b92c84f66f 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -310,9 +310,9 @@ func (m *GSOCEphemeralMode) handlePublisher(ctx context.Context, peer p2p.Peer, if err != nil { if errors.Is(err, io.EOF) { m.logger.Info("publisher stream EOF", "peer", peer.Address) - } else { - return fmt.Errorf("read publisher message: %w", err) + return nil } + return fmt.Errorf("read publisher message: %w", err) } m.logger.Info("publisher message received", "peer", peer.Address, "size", len(rawMsg)) From b7783ceff9a6ca2ba91ae157cf8725858ecc28f8 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 14:38:47 +0200 Subject: [PATCH 21/51] fix: p2p ping --- pkg/pubsub/mode.go | 16 ++++++++++++++++ pkg/pubsub/ws.go | 3 +++ 2 files changed, 19 insertions(+) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 3b92c84f66f..c9385ee9b29 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "sync" + "time" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" @@ -55,6 +56,10 @@ const ( // Message types (Broker → Subscriber) MsgTypeHandshake byte = 0x01 MsgTypeData byte = 0x02 + MsgTypePing byte = 0x03 + + // Ping interval for keeping p2p streams alive. + streamPingInterval = 30 * time.Second ) // GSOCEphemeralMode implements Mode for GSOC ephemeral messaging. @@ -195,6 +200,10 @@ func (m *GSOCEphemeralMode) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) } switch typeBuf[0] { + case MsgTypePing: + m.logger.Debug("received ping from broker") + return nil, nil + case MsgTypeHandshake: socID := make([]byte, IDSize) if _, err := io.ReadFull(stream, socID); err != nil { @@ -337,6 +346,8 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar m.mu.Unlock() go func() { + ticker := time.NewTicker(streamPingInterval) + defer ticker.Stop() for { select { case <-connCtx.Done(): @@ -346,6 +357,11 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar cancel() return } + case <-ticker.C: + if err := writeRaw(stream, []byte{MsgTypePing}); err != nil { + cancel() + return + } } } }() diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index 78f9fdbfdeb..dfe0e3baf94 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -86,6 +86,9 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l } wsPayload, err := mode.ReadBrokerMessage(sc.Stream) + if wsPayload == nil { + continue + } if err != nil { if ctx.Err() == nil { logger.Info("pubsub ws: read broker message failed", "error", err) From 92ba5ed94ccc79d43150aafa56fd303116cde957 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 15:16:33 +0200 Subject: [PATCH 22/51] chore: debugging on broker side --- pkg/pubsub/mode.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index c9385ee9b29..6f9fe5a7746 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -354,9 +354,11 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar return case msg := <-sub.outCh: if err := writeRaw(stream, msg); err != nil { + m.logger.Info("broker write to subscriber failed", "peer", sub.overlay, "error", err) cancel() return } + m.logger.Info("broker wrote to subscriber", "peer", sub.overlay, "size", len(msg)) case <-ticker.C: if err := writeRaw(stream, []byte{MsgTypePing}); err != nil { cancel() From d2910b985e55154c7cbc34af721810e26cbdae0e Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 15:22:37 +0200 Subject: [PATCH 23/51] chore: debug --- pkg/pubsub/mode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 6f9fe5a7746..c3dba213b01 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -295,6 +295,7 @@ func (m *GSOCEphemeralMode) handleSubscriber(ctx context.Context, peer p2p.Peer, func (m *GSOCEphemeralMode) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p.Stream, headers p2p.Headers) error { if err := m.validatePublisher(headers); err != nil { + m.logger.Debug("invalid publisher headers", "error", err) _ = stream.Reset() return err } From 267017635f2681ca8a250254477563fa6c98885c Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Wed, 22 Apr 2026 15:57:30 +0200 Subject: [PATCH 24/51] chore: add logs for debugging --- pkg/pubsub/mode.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index c3dba213b01..ab64d0ecb05 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -267,7 +267,7 @@ func (m *GSOCEphemeralMode) broadcast(rawMsg []byte) { // HandleBroker handles an incoming broker-side stream, dispatching to publisher or subscriber handling. func (m *GSOCEphemeralMode) HandleBroker(ctx context.Context, peer p2p.Peer, stream p2p.Stream, headers p2p.Headers) error { rwBytes := headers[HeaderReadWrite] - m.logger.Debug("reading rw header", "peer", peer.Address) + m.logger.Info("broker stream opened", "peer", peer.Address, "topic", m.TopicAddress(), "rw", rwBytes) if len(rwBytes) != 1 { _ = stream.Reset() return ErrWrongHeaders @@ -275,7 +275,7 @@ func (m *GSOCEphemeralMode) HandleBroker(ctx context.Context, peer p2p.Peer, str if rwBytes[0] == 1 { return m.handlePublisher(ctx, peer, stream, headers) } - m.logger.Debug("handling as subscriber", "peer", peer.Address) + m.logger.Info("handling as subscriber", "peer", peer.Address) return m.handleSubscriber(ctx, peer, stream) } @@ -284,7 +284,7 @@ func (m *GSOCEphemeralMode) handleSubscriber(ctx context.Context, peer p2p.Peer, defer cancel() defer unregister() - m.logger.Debug("subscriber connected", "peer", peer.Address, "topic", m.TopicAddress()) + m.logger.Info("subscriber connected", "peer", peer.Address, "topic", m.TopicAddress()) <-subCtx.Done() if errors.Is(subCtx.Err(), context.Canceled) { @@ -294,17 +294,20 @@ func (m *GSOCEphemeralMode) handleSubscriber(ctx context.Context, peer p2p.Peer, } func (m *GSOCEphemeralMode) handlePublisher(ctx context.Context, peer p2p.Peer, stream p2p.Stream, headers p2p.Headers) error { + m.logger.Info("publisher handler entered", "peer", peer.Address, "topic", m.TopicAddress()) + if err := m.validatePublisher(headers); err != nil { - m.logger.Debug("invalid publisher headers", "error", err) + m.logger.Info("publisher validation failed", "peer", peer.Address, "error", err) _ = stream.Reset() return err } + m.logger.Info("publisher validated", "peer", peer.Address, "gsoc_id", fmt.Sprintf("%x", m.gsocID), "gsoc_owner", fmt.Sprintf("%x", m.gsocOwner)) partCtx, cancel, unregister := m.registerSubscriber(ctx, peer.Address, stream) defer cancel() defer unregister() - m.logger.Debug("publisher connected", "peer", peer.Address, "topic", m.TopicAddress()) + m.logger.Info("publisher connected, starting read loop", "peer", peer.Address, "topic", m.TopicAddress(), "subscribers", m.SubscriberCount()) for { select { @@ -316,12 +319,14 @@ func (m *GSOCEphemeralMode) handlePublisher(ctx context.Context, peer p2p.Peer, default: } + m.logger.Info("waiting for publisher message", "peer", peer.Address) rawMsg, err := m.ReadPublisherMessage(stream) if err != nil { if errors.Is(err, io.EOF) { m.logger.Info("publisher stream EOF", "peer", peer.Address) return nil } + m.logger.Info("publisher read error", "peer", peer.Address, "error", err) return fmt.Errorf("read publisher message: %w", err) } From ac843ad4ed083ad1841a14814f8839e3a4aff511 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 16:00:30 +0200 Subject: [PATCH 25/51] chore: debug instead of info log --- pkg/pubsub/pubsub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index d0583b06c40..1800912f956 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -185,7 +185,7 @@ func (s *Service) Topics() []TopicInfo { // brokerHandler handles incoming streams on the broker side. func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.Stream) error { - s.logger.Debug("broker handler invoked", "peer", peer.Address) + s.logger.Info("broker handler invoked", "peer", peer.Address) if !s.brokerMode { _ = stream.Reset() return ErrBrokerDisabled From af506a2eb79149b89c9848106d23d46764a643a5 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 22 Apr 2026 16:10:58 +0200 Subject: [PATCH 26/51] chore: message check --- pkg/pubsub/mode.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index ab64d0ecb05..c049e979f2f 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -186,6 +186,7 @@ func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, err copy(socData[IDSize+SigSize+SpanSize:], payload) if !soc.Valid(swarm.NewChunk(m.topicAddress, socData)) { + m.logger.Debug("soc validation failed", "topicAddress", m.topicAddress, "socData", socData) return nil, ErrInvalidSignature } @@ -416,6 +417,7 @@ func (m *GSOCEphemeralMode) setGsocParams(gsocOwner, gsocID []byte) { // Verify got socId and address match with topicaddress addr, err := soc.CreateAddress(gsocID, gsocOwner) if err != nil || !bytes.Equal(addr.Bytes(), m.topicAddress.Bytes()) { + m.logger.Debug("gsoc params verification failed", "err", err, "addr", addr, "topicAddress", m.topicAddress) return } From dd4172dbe63e243d241fafb6577c2461e1916cfd Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Wed, 22 Apr 2026 16:08:46 +0200 Subject: [PATCH 27/51] fix: soc sig validation --- pkg/pubsub/mode.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index c049e979f2f..b41c99082af 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -177,16 +177,20 @@ func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, err return nil, err } - // Construct SOC chunk with the known topic address: [ID (32B)][sig (65B)][span (4B)][payload] - // and validate whether message is valid + // Construct SOC chunk: [ID (32B)][sig (65B)][span (4B)][payload] + // The chunk address for SOC validation is hash(gsocID, gsocOwner), not the topic address. socData := make([]byte, IDSize+SigSize+SpanSize+int(span)) copy(socData, m.gsocID) copy(socData[IDSize:], sig) copy(socData[IDSize+SigSize:], spanBytes) copy(socData[IDSize+SigSize+SpanSize:], payload) - if !soc.Valid(swarm.NewChunk(m.topicAddress, socData)) { - m.logger.Debug("soc validation failed", "topicAddress", m.topicAddress, "socData", socData) + socAddr, err := soc.CreateAddress(m.gsocID, m.gsocOwner) + if err != nil { + return nil, fmt.Errorf("pubsub: compute SOC address: %w", err) + } + + if !soc.Valid(swarm.NewChunk(socAddr, socData)) { return nil, ErrInvalidSignature } From 5633c6336ccb0c736e1fefb7f603c9811b74fd6d Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Wed, 22 Apr 2026 16:19:33 +0200 Subject: [PATCH 28/51] fix: chunk span calc --- pkg/pubsub/mode.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index b41c99082af..9080461d4b5 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -177,13 +177,18 @@ func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, err return nil, err } - // Construct SOC chunk: [ID (32B)][sig (65B)][span (4B)][payload] - // The chunk address for SOC validation is hash(gsocID, gsocOwner), not the topic address. - socData := make([]byte, IDSize+SigSize+SpanSize+int(span)) + // Construct SOC chunk: [ID (32B)][sig (65B)][span (8B)][payload] + // The wire format uses a 4-byte uint32 span, but cac.NewWithDataSpan expects the + // standard 8-byte little-endian span (swarm.SpanSize). Zero-pad to 8 bytes here; + // since MaxPayload <= 4096 the upper 4 bytes are always zero. + span8 := make([]byte, swarm.SpanSize) + copy(span8, spanBytes) // copy 4 bytes; upper 4 remain zero + + socData := make([]byte, IDSize+SigSize+swarm.SpanSize+int(span)) copy(socData, m.gsocID) copy(socData[IDSize:], sig) - copy(socData[IDSize+SigSize:], spanBytes) - copy(socData[IDSize+SigSize+SpanSize:], payload) + copy(socData[IDSize+SigSize:], span8) + copy(socData[IDSize+SigSize+swarm.SpanSize:], payload) socAddr, err := soc.CreateAddress(m.gsocID, m.gsocOwner) if err != nil { From f14697b13a6182360156cd05c5780e92e05f4c62 Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Wed, 22 Apr 2026 16:35:37 +0200 Subject: [PATCH 29/51] fix: span size 8 --- pkg/pubsub/mode.go | 14 ++++---------- pkg/pubsub/pubsub.go | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 9080461d4b5..084482e74ee 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -170,7 +170,7 @@ func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, err if _, err := io.ReadFull(stream, spanBytes); err != nil { return nil, err } - span := min(binary.LittleEndian.Uint32(spanBytes), MaxPayload) + span := min(binary.LittleEndian.Uint64(spanBytes), MaxPayload) payload := make([]byte, span) if _, err := io.ReadFull(stream, payload); err != nil { @@ -178,17 +178,11 @@ func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, err } // Construct SOC chunk: [ID (32B)][sig (65B)][span (8B)][payload] - // The wire format uses a 4-byte uint32 span, but cac.NewWithDataSpan expects the - // standard 8-byte little-endian span (swarm.SpanSize). Zero-pad to 8 bytes here; - // since MaxPayload <= 4096 the upper 4 bytes are always zero. - span8 := make([]byte, swarm.SpanSize) - copy(span8, spanBytes) // copy 4 bytes; upper 4 remain zero - - socData := make([]byte, IDSize+SigSize+swarm.SpanSize+int(span)) + socData := make([]byte, IDSize+SigSize+SpanSize+int(span)) copy(socData, m.gsocID) copy(socData[IDSize:], sig) - copy(socData[IDSize+SigSize:], span8) - copy(socData[IDSize+SigSize+swarm.SpanSize:], payload) + copy(socData[IDSize+SigSize:], spanBytes) + copy(socData[IDSize+SigSize+SpanSize:], payload) socAddr, err := soc.CreateAddress(m.gsocID, m.gsocOwner) if err != nil { diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 1800912f956..c221bd6f0f1 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -32,7 +32,7 @@ const ( ModeGSOCEphemeral ModeID = 1 // Wire format sizes - SpanSize = 4 // pubsub span: uint32 little-endian + SpanSize = swarm.SpanSize // pubsub span: 8-byte little-endian uint64 (matches bee-js Span.LENGTH) MaxPayload = swarm.ChunkSize SigSize = swarm.SocSignatureSize IDSize = swarm.HashSize From 991ce657a0d01816789f06c6bffdcb6cb9cf4aa5 Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Thu, 23 Apr 2026 12:03:00 +0200 Subject: [PATCH 30/51] fix: swap pubsub ws upgrade and connect order --- pkg/api/pubsub.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index a6821ea24a9..13b5e08a2dc 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -95,16 +95,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { return } - // Connect to broker peer - ctx, cancel := context.WithCancel(context.Background()) - mode, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) - if err != nil { - cancel() - logger.Info("pubsub connect failed", "error", err) - jsonhttp.InternalServerError(w, "pubsub connect failed") - return - } - // Upgrade to WebSocket + // upgrade to WebSocket upgrader := websocket.Upgrader{ ReadBufferSize: swarm.ChunkWithSpanSize, WriteBufferSize: swarm.ChunkWithSpanSize, @@ -114,18 +105,25 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { logger.Info("upgrading to websocket") conn, err := upgrader.Upgrade(w, r, nil) if err != nil { - cancel() - if sc := mode.GetSubscriberConn(); sc != nil { - _ = sc.Stream.Close() - } - logger.Info("websocket upgrade failed", "error", err) - logger.Error(nil, "websocket upgrade failed") - // NOTE: Upgrade() hijacks the connection before returning an error, + // Upgrade() hijacks the connection before returning an error, // so do NOT write an HTTP response here. + logger.Info("websocket upgrade failed", "error", err) return } logger.Info("websocket upgrade successful") + // open the p2p stream to the broker. + ctx, cancel := context.WithCancel(context.Background()) + mode, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) + if err != nil { + cancel() + logger.Info("pubsub connect failed", "error", err) + _ = conn.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "pubsub connect failed")) + _ = conn.Close() + return + } + pingPeriod := headers.KeepAlive * time.Second if pingPeriod == 0 { pingPeriod = time.Minute From 0272fff6920309b649ddb37b620503cf5d3875df Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Thu, 23 Apr 2026 13:39:50 +0200 Subject: [PATCH 31/51] fix: revert ws and svc connect order and cancel subscriberConn if presend during create --- pkg/api/pubsub.go | 28 +++++++++++++++------------- pkg/pubsub/mode.go | 5 +++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 13b5e08a2dc..2e83e682b3a 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -95,7 +95,16 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { return } - // upgrade to WebSocket + // Connect to broker peer + ctx, cancel := context.WithCancel(context.Background()) + mode, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) + if err != nil { + cancel() + logger.Info("pubsub connect failed", "error", err) + jsonhttp.InternalServerError(w, "pubsub connect failed") + return + } + // Upgrade to WebSocket upgrader := websocket.Upgrader{ ReadBufferSize: swarm.ChunkWithSpanSize, WriteBufferSize: swarm.ChunkWithSpanSize, @@ -107,23 +116,16 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { if err != nil { // Upgrade() hijacks the connection before returning an error, // so do NOT write an HTTP response here. + cancel() + if sc := mode.GetSubscriberConn(); sc != nil { + _ = sc.Stream.Close() + } logger.Info("websocket upgrade failed", "error", err) + logger.Error(nil, "websocket upgrade failed") return } logger.Info("websocket upgrade successful") - // open the p2p stream to the broker. - ctx, cancel := context.WithCancel(context.Background()) - mode, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) - if err != nil { - cancel() - logger.Info("pubsub connect failed", "error", err) - _ = conn.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "pubsub connect failed")) - _ = conn.Close() - return - } - pingPeriod := headers.KeepAlive * time.Second if pingPeriod == 0 { pingPeriod = time.Minute diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 084482e74ee..8753100f375 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -390,6 +390,11 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar // CreateSubscriberConn creates and stores the subscriber-side connection in this mode. func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) { m.mu.Lock() + + if m.subscriberConn != nil { + m.subscriberConn.Cancel() + } + m.subscriberConn = &SubscriberConn{ Stream: stream, Overlay: overlay, From 70de89ccf78821b22b09b1a8793b899b92570f5b Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Thu, 23 Apr 2026 14:15:18 +0200 Subject: [PATCH 32/51] fix: unregister delete overlay --- pkg/pubsub/mode.go | 4 +++- pkg/topology/kademlia/kademlia.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 8753100f375..05b819e7cf9 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -380,7 +380,9 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar unregister := func() { m.mu.Lock() - delete(m.subscribers, overlayKey) + if m.subscribers[overlayKey] == sub { + delete(m.subscribers, overlayKey) + } m.mu.Unlock() } diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index 78451a986c2..87919438ff4 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -991,7 +991,7 @@ func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma []ma.Multiaddr case errors.Is(err, p2p.ErrUnsupportedAddresses): return err case err != nil: - k.logger.Info("could not connect to peer", "peer_address", peer, "error", err) + k.logger.Debug("could not connect to peer", "peer_address", peer, "error", err) retryTime := time.Now().Add(k.opt.TimeToRetry) var e *p2p.ConnectionBackoffError From 775bcb0412f77f5b2f59e23e188cbf1814401728 Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 23 Apr 2026 14:47:01 +0200 Subject: [PATCH 33/51] feat: allow light node mode --- pkg/p2p/libp2p/libp2p.go | 15 +++++++++++++-- pkg/pubsub/pubsub.go | 6 +++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 462322c0d0e..2c4d672c8a9 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -965,7 +965,18 @@ func buildHostAddress(peerID libp2ppeer.ID) (ma.Multiaddr, error) { return ma.NewMultiaddr(fmt.Sprintf("/p2p/%s", peerID.String())) } -func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *bzz.Address, err error) { +func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { + return s.connect(ctx, addrs, false) +} + +// ConnectAllowLight behaves like Connect but does not reject the peer +// if it identifies itself as a light node. Intended for protocols such +// as pubsub where broker peers may operate in light-node mode. +func (s *Service) ConnectAllowLight(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { + return s.connect(ctx, addrs, true) +} + +func (s *Service) connect(ctx context.Context, addrs []ma.Multiaddr, allowLight bool) (address *bzz.Address, err error) { loggerV1 := s.logger.V(1).Register() defer func() { @@ -1099,7 +1110,7 @@ func (s *Service) Connect(ctx context.Context, addrs []ma.Multiaddr) (address *b return nil, fmt.Errorf("handshake: %w", err) } - if !i.FullNode { + if !i.FullNode && !allowLight { _ = handshakeStream.Reset() _ = s.host.Network().ClosePeer(info.ID) return nil, p2p.ErrDialLightNode diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index c221bd6f0f1..fb4a7df9547 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -10,6 +10,7 @@ import ( "fmt" "sync" + "github.com/ethersphere/bee/v2/pkg/bzz" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" @@ -81,6 +82,9 @@ type TopicModeKey struct { type P2P interface { p2p.Service p2p.Streamer + // ConnectAllowLight dials a peer, accepting it even if it identifies + // itself as a light node (broker peers may run in light-node mode). + ConnectAllowLight(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) } // Service is the pubsub protocol service. @@ -127,7 +131,7 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr } s.logger.Info("connecting to broker peer", "underlay", underlay) - bzzAddr, err := s.p2p.Connect(ctx, []ma.Multiaddr{underlay}) + bzzAddr, err := s.p2p.ConnectAllowLight(ctx, []ma.Multiaddr{underlay}) if err != nil && !errors.Is(err, p2p.ErrAlreadyConnected) { return nil, fmt.Errorf("connect to peer: %w", err) } From 0a7f7b75810a39958dff751ff858d62b14d706b5 Mon Sep 17 00:00:00 2001 From: Balint Ujvari Date: Thu, 23 Apr 2026 14:52:18 +0200 Subject: [PATCH 34/51] fix: remove and create sub conn race conditions --- pkg/api/pubsub.go | 4 +--- pkg/pubsub/mode.go | 18 +++++++++++------- pkg/pubsub/pubsub.go | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 2e83e682b3a..69f00df0738 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -136,10 +136,8 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { s.wsWg.Add(1) go func() { pubsub.ListeningWs(ctx, conn, pubsub.WsOptions{PingPeriod: pingPeriod, Cancel: cancel}, logger, mode, isPublisher) + cancel() _ = conn.Close() - if sc := mode.GetSubscriberConn(); sc != nil { - sc.Cancel() - } s.wsWg.Done() }() } diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 05b819e7cf9..d89211b4026 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -39,9 +39,9 @@ type Mode interface { // Subscriber side - outbound connection to broker Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) - CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) + CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) *SubscriberConn GetSubscriberConn() *SubscriberConn - RemoveSubscriberConn() + RemoveSubscriberConn(conn *SubscriberConn) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) // Broker side - handles incoming streams (publisher and subscriber) @@ -390,19 +390,21 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar } // CreateSubscriberConn creates and stores the subscriber-side connection in this mode. -func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) { +func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) *SubscriberConn { m.mu.Lock() + defer m.mu.Unlock() if m.subscriberConn != nil { m.subscriberConn.Cancel() } - m.subscriberConn = &SubscriberConn{ + sc := &SubscriberConn{ Stream: stream, Overlay: overlay, Cancel: cancel, } - m.mu.Unlock() + m.subscriberConn = sc + return sc } // GetSubscriberConn returns the subscriber-side connection, or nil. @@ -413,9 +415,11 @@ func (m *GSOCEphemeralMode) GetSubscriberConn() *SubscriberConn { } // RemoveSubscriberConn clears the subscriber-side connection. -func (m *GSOCEphemeralMode) RemoveSubscriberConn() { +func (m *GSOCEphemeralMode) RemoveSubscriberConn(conn *SubscriberConn) { m.mu.Lock() - m.subscriberConn = nil + if m.subscriberConn == conn { + m.subscriberConn = nil + } m.mu.Unlock() } diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index fb4a7df9547..e04599e5862 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -144,12 +144,12 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr } connCtx, cancel := context.WithCancel(ctx) - m.CreateSubscriberConn(stream, bzzAddr.Overlay, cancel) + sc := m.CreateSubscriberConn(stream, bzzAddr.Overlay, cancel) go func() { <-connCtx.Done() _ = stream.FullClose() - m.RemoveSubscriberConn() + m.RemoveSubscriberConn(sc) }() return m, nil From 2242743733c48ca347bd3b9ae7e2923045e89af1 Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 23 Apr 2026 18:32:16 +0200 Subject: [PATCH 35/51] fix: multiple ws on the same topic --- pkg/pubsub/mode.go | 102 ++++++++++++++++++++++++++++++++++++++----- pkg/pubsub/pubsub.go | 25 +++++++---- pkg/pubsub/ws.go | 36 +++++++-------- 3 files changed, 123 insertions(+), 40 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index d89211b4026..6b8e163169f 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -39,7 +39,7 @@ type Mode interface { // Subscriber side - outbound connection to broker Connect(ctx context.Context, p p2p.Streamer, overlay swarm.Address, opts ConnectOptions) (p2p.Stream, error) - CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) *SubscriberConn + CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address) *SubscriberConn GetSubscriberConn() *SubscriberConn RemoveSubscriberConn(conn *SubscriberConn) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) @@ -82,11 +82,65 @@ type brokerSubscriber struct { handshakeHappened bool } -// SubscriberConn represents the subscriber-side connection to a broker. +// SubscriberConn represents the shared subscriber-side p2p stream to a broker. +// Multiple WebSocket sessions can attach to one SubscriberConn via the mux. type SubscriberConn struct { Stream p2p.Stream Overlay swarm.Address - Cancel context.CancelFunc + + refs int // number of active WS sessions; protected by the owning mode's mu + writeMu sync.Mutex + subsMu sync.Mutex + subs map[uint64]chan []byte + nextID uint64 +} + +// Subscribe registers a new WS session and returns its per-session message channel. +func (sc *SubscriberConn) Subscribe() (uint64, <-chan []byte) { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + id := sc.nextID + sc.nextID++ + ch := make(chan []byte, 16) + sc.subs[id] = ch + return id, ch +} + +// Unsubscribe removes the WS session channel and closes it. +func (sc *SubscriberConn) Unsubscribe(id uint64) { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + if ch, ok := sc.subs[id]; ok { + close(ch) + delete(sc.subs, id) + } +} + +func (sc *SubscriberConn) broadcast(msg []byte) { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + for _, ch := range sc.subs { + select { + case ch <- msg: + default: + } + } +} + +func (sc *SubscriberConn) closeAll() { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + for id, ch := range sc.subs { + close(ch) + delete(sc.subs, id) + } +} + +// WriteToStream serializes concurrent writes from multiple WS sessions. +func (sc *SubscriberConn) WriteToStream(data []byte) error { + sc.writeMu.Lock() + defer sc.writeMu.Unlock() + return writeRaw(sc.Stream, data) } var _ Mode = (*GSOCEphemeralMode)(nil) @@ -389,24 +443,46 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar return connCtx, cancel, unregister } -// CreateSubscriberConn creates and stores the subscriber-side connection in this mode. -func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address, cancel context.CancelFunc) *SubscriberConn { +// CreateSubscriberConn returns the existing SubscriberConn for this topic if one is active, +// incrementing its ref count so the shared stream stays open. When no conn exists yet, +// a new one is created and a single mux goroutine is started to fan out broker messages. +func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swarm.Address) *SubscriberConn { m.mu.Lock() defer m.mu.Unlock() if m.subscriberConn != nil { - m.subscriberConn.Cancel() + m.subscriberConn.refs++ + return m.subscriberConn } sc := &SubscriberConn{ Stream: stream, Overlay: overlay, - Cancel: cancel, + refs: 1, + subs: make(map[uint64]chan []byte), } m.subscriberConn = sc + go m.runMux(sc, stream) return sc } +// runMux reads broker messages from the shared p2p stream and broadcasts each to all +// registered WS sessions. It exits when the stream closes or returns an error. +func (m *GSOCEphemeralMode) runMux(sc *SubscriberConn, stream p2p.Stream) { + defer sc.closeAll() + for { + msg, err := m.ReadBrokerMessage(stream) + if err != nil { + m.logger.Debug("pubsub mux: stream error, stopping", "error", err) + return + } + if msg == nil { + continue + } + sc.broadcast(msg) + } +} + // GetSubscriberConn returns the subscriber-side connection, or nil. func (m *GSOCEphemeralMode) GetSubscriberConn() *SubscriberConn { m.mu.RLock() @@ -414,13 +490,19 @@ func (m *GSOCEphemeralMode) GetSubscriberConn() *SubscriberConn { return m.subscriberConn } -// RemoveSubscriberConn clears the subscriber-side connection. +// RemoveSubscriberConn decrements the ref count for the connection. +// When the last WS session exits, it closes the stream, stopping the mux goroutine. func (m *GSOCEphemeralMode) RemoveSubscriberConn(conn *SubscriberConn) { m.mu.Lock() - if m.subscriberConn == conn { + defer m.mu.Unlock() + if m.subscriberConn != conn { + return + } + conn.refs-- + if conn.refs <= 0 { m.subscriberConn = nil + _ = conn.Stream.FullClose() } - m.mu.Unlock() } // setGsocParams sets the GSOC recurring parameters so that messages don't need to include them. diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index e04599e5862..2e782e7eec5 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -137,18 +137,25 @@ func (s *Service) Connect(ctx context.Context, underlay ma.Multiaddr, topicAddr } s.logger.Info("connected to broker peer", "overlay", bzzAddr.Overlay) - stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) - if err != nil { - s.logger.Error(err, "open stream failed") - return nil, fmt.Errorf("open stream: %w", err) + var sc *SubscriberConn + if existing := m.GetSubscriberConn(); existing != nil { + // Reuse the existing p2p stream — no new broker-side stream, just bump the ref count. + sc = m.CreateSubscriberConn(existing.Stream, bzzAddr.Overlay) + } else { + stream, err := m.Connect(ctx, s.p2p, bzzAddr.Overlay, opts) + if err != nil { + s.logger.Error(err, "open stream failed") + return nil, fmt.Errorf("open stream: %w", err) + } + sc = m.CreateSubscriberConn(stream, bzzAddr.Overlay) + if sc.Stream != stream { + // Race: another goroutine created the conn between our check and create. + _ = stream.FullClose() + } } - connCtx, cancel := context.WithCancel(ctx) - sc := m.CreateSubscriberConn(stream, bzzAddr.Overlay, cancel) - go func() { - <-connCtx.Done() - _ = stream.FullClose() + <-ctx.Done() m.RemoveSubscriberConn(sc) }() diff --git a/pkg/pubsub/ws.go b/pkg/pubsub/ws.go index dfe0e3baf94..1cc65793ac4 100644 --- a/pkg/pubsub/ws.go +++ b/pkg/pubsub/ws.go @@ -24,6 +24,7 @@ type WsOptions struct { // and writes raw messages to the p2p stream. func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, logger log.Logger, mode Mode, isPublisher bool) { sc := mode.GetSubscriberConn() + subID, msgCh := sc.Subscribe() var ( ticker = time.NewTicker(options.PingPeriod) writeDeadline = options.PingPeriod + time.Second @@ -66,7 +67,7 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l if isPublisher { logger.Info("pubsub ws: publisher message from ws", "type", msgType, "size", len(p)) - if err := writeRaw(sc.Stream, p); err != nil { + if err := sc.WriteToStream(p); err != nil { logger.Info("pubsub ws: write to p2p stream failed", "error", err) break } @@ -75,33 +76,26 @@ func ListeningWs(ctx context.Context, conn *websocket.Conn, options WsOptions, l options.Cancel() }() - // Read from p2p stream (Broker messages) and forward to WebSocket. + // Forward mux-delivered broker messages to this WebSocket session. go func() { + defer sc.Unsubscribe(subID) for { select { case <-ctx.Done(): logger.Info("pubsub ws: p2p reader context done") return - default: - } - - wsPayload, err := mode.ReadBrokerMessage(sc.Stream) - if wsPayload == nil { - continue - } - if err != nil { - if ctx.Err() == nil { - logger.Info("pubsub ws: read broker message failed", "error", err) + case msg, ok := <-msgCh: + if !ok { + logger.Info("pubsub ws: mux channel closed") + options.Cancel() + return + } + logger.Info("pubsub ws: forwarding broker message to ws", "size", len(msg)) + if err := conn.WriteMessage(websocket.BinaryMessage, msg); err != nil { + logger.Info("pubsub ws: write to ws failed", "error", err) + options.Cancel() + return } - options.Cancel() - return - } - - logger.Info("pubsub ws: forwarding broker message to ws", "size", len(wsPayload)) - if err := conn.WriteMessage(websocket.BinaryMessage, wsPayload); err != nil { - logger.Info("pubsub ws: write to ws failed", "error", err) - options.Cancel() - return } } }() From 3998eb88ea328ff43972387dc6ebaf3fc186eecd Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 16:49:22 +0200 Subject: [PATCH 36/51] refactor: remove unnecessary param in runMux --- pkg/pubsub/mode.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 6b8e163169f..f574b9dde38 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -462,14 +462,14 @@ func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swar subs: make(map[uint64]chan []byte), } m.subscriberConn = sc - go m.runMux(sc, stream) + go m.runMux(stream) return sc } // runMux reads broker messages from the shared p2p stream and broadcasts each to all // registered WS sessions. It exits when the stream closes or returns an error. -func (m *GSOCEphemeralMode) runMux(sc *SubscriberConn, stream p2p.Stream) { - defer sc.closeAll() +func (m *GSOCEphemeralMode) runMux(stream p2p.Stream) { + defer m.subscriberConn.closeAll() for { msg, err := m.ReadBrokerMessage(stream) if err != nil { @@ -479,7 +479,7 @@ func (m *GSOCEphemeralMode) runMux(sc *SubscriberConn, stream p2p.Stream) { if msg == nil { continue } - sc.broadcast(msg) + m.subscriberConn.broadcast(msg) } } From da3bb7e757cfc295b0c19e76eb3a6d29b4ce929f Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 16:53:15 +0200 Subject: [PATCH 37/51] refactor: move connection related struct defs and methods to service --- pkg/pubsub/mode.go | 70 -------------------------------------------- pkg/pubsub/pubsub.go | 70 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index f574b9dde38..95191011655 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -73,76 +73,6 @@ type GSOCEphemeralMode struct { subscriberConn *SubscriberConn } -// brokerSubscriber holds a subscriber's stream and outgoing message channel. -type brokerSubscriber struct { - overlay swarm.Address - stream p2p.Stream - outCh chan []byte - cancel context.CancelFunc - handshakeHappened bool -} - -// SubscriberConn represents the shared subscriber-side p2p stream to a broker. -// Multiple WebSocket sessions can attach to one SubscriberConn via the mux. -type SubscriberConn struct { - Stream p2p.Stream - Overlay swarm.Address - - refs int // number of active WS sessions; protected by the owning mode's mu - writeMu sync.Mutex - subsMu sync.Mutex - subs map[uint64]chan []byte - nextID uint64 -} - -// Subscribe registers a new WS session and returns its per-session message channel. -func (sc *SubscriberConn) Subscribe() (uint64, <-chan []byte) { - sc.subsMu.Lock() - defer sc.subsMu.Unlock() - id := sc.nextID - sc.nextID++ - ch := make(chan []byte, 16) - sc.subs[id] = ch - return id, ch -} - -// Unsubscribe removes the WS session channel and closes it. -func (sc *SubscriberConn) Unsubscribe(id uint64) { - sc.subsMu.Lock() - defer sc.subsMu.Unlock() - if ch, ok := sc.subs[id]; ok { - close(ch) - delete(sc.subs, id) - } -} - -func (sc *SubscriberConn) broadcast(msg []byte) { - sc.subsMu.Lock() - defer sc.subsMu.Unlock() - for _, ch := range sc.subs { - select { - case ch <- msg: - default: - } - } -} - -func (sc *SubscriberConn) closeAll() { - sc.subsMu.Lock() - defer sc.subsMu.Unlock() - for id, ch := range sc.subs { - close(ch) - delete(sc.subs, id) - } -} - -// WriteToStream serializes concurrent writes from multiple WS sessions. -func (sc *SubscriberConn) WriteToStream(data []byte) error { - sc.writeMu.Lock() - defer sc.writeMu.Unlock() - return writeRaw(sc.Stream, data) -} - var _ Mode = (*GSOCEphemeralMode)(nil) func NewGSOCEphemeralMode(topicAddress []byte, logger log.Logger) *GSOCEphemeralMode { diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 2e782e7eec5..6a8ccc90079 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -261,3 +261,73 @@ func writeRaw(stream p2p.Stream, data []byte) error { } return nil } + +// brokerSubscriber holds a subscriber's stream and outgoing message channel. +type brokerSubscriber struct { + overlay swarm.Address + stream p2p.Stream + outCh chan []byte + cancel context.CancelFunc + handshakeHappened bool +} + +// SubscriberConn represents the shared subscriber-side p2p stream to a broker. +// Multiple WebSocket sessions can attach to one SubscriberConn via the mux. +type SubscriberConn struct { + Stream p2p.Stream + Overlay swarm.Address + + refs int // number of active WS sessions; protected by the owning mode's mu + writeMu sync.Mutex + subsMu sync.Mutex + subs map[uint64]chan []byte + nextID uint64 +} + +// Subscribe registers a new WS session and returns its per-session message channel. +func (sc *SubscriberConn) Subscribe() (uint64, <-chan []byte) { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + id := sc.nextID + sc.nextID++ + ch := make(chan []byte, 16) + sc.subs[id] = ch + return id, ch +} + +// Unsubscribe removes the WS session channel and closes it. +func (sc *SubscriberConn) Unsubscribe(id uint64) { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + if ch, ok := sc.subs[id]; ok { + close(ch) + delete(sc.subs, id) + } +} + +func (sc *SubscriberConn) broadcast(msg []byte) { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + for _, ch := range sc.subs { + select { + case ch <- msg: + default: + } + } +} + +func (sc *SubscriberConn) closeAll() { + sc.subsMu.Lock() + defer sc.subsMu.Unlock() + for id, ch := range sc.subs { + close(ch) + delete(sc.subs, id) + } +} + +// WriteToStream serializes concurrent writes from multiple WS sessions. +func (sc *SubscriberConn) WriteToStream(data []byte) error { + sc.writeMu.Lock() + defer sc.writeMu.Unlock() + return writeRaw(sc.Stream, data) +} From 510d9a68d455ecc2ee1c6dee8052ece73a2d4cae Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 17:24:21 +0200 Subject: [PATCH 38/51] refactor: subtract the pinging service from mode --- pkg/pubsub/mode.go | 38 +++++++---------------------------- pkg/pubsub/pubsub.go | 47 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 32 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 95191011655..de013952754 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -12,7 +12,6 @@ import ( "fmt" "io" "sync" - "time" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" @@ -56,10 +55,6 @@ const ( // Message types (Broker → Subscriber) MsgTypeHandshake byte = 0x01 MsgTypeData byte = 0x02 - MsgTypePing byte = 0x03 - - // Ping interval for keeping p2p streams alive. - streamPingInterval = 30 * time.Second ) // GSOCEphemeralMode implements Mode for GSOC ephemeral messaging. @@ -187,11 +182,13 @@ func (m *GSOCEphemeralMode) ReadBrokerMessage(stream p2p.Stream) ([]byte, error) return nil, err } - switch typeBuf[0] { - case MsgTypePing: - m.logger.Debug("received ping from broker") + if handled, err := readServiceMessage(typeBuf[0]); err != nil { + return nil, err + } else if handled { return nil, nil + } + switch typeBuf[0] { case MsgTypeHandshake: socID := make([]byte, IDSize) if _, err := io.ReadFull(stream, socID); err != nil { @@ -339,28 +336,7 @@ func (m *GSOCEphemeralMode) registerSubscriber(ctx context.Context, overlay swar m.subscribers[overlayKey] = sub m.mu.Unlock() - go func() { - ticker := time.NewTicker(streamPingInterval) - defer ticker.Stop() - for { - select { - case <-connCtx.Done(): - return - case msg := <-sub.outCh: - if err := writeRaw(stream, msg); err != nil { - m.logger.Info("broker write to subscriber failed", "peer", sub.overlay, "error", err) - cancel() - return - } - m.logger.Info("broker wrote to subscriber", "peer", sub.overlay, "size", len(msg)) - case <-ticker.C: - if err := writeRaw(stream, []byte{MsgTypePing}); err != nil { - cancel() - return - } - } - } - }() + startBrokerWriter(connCtx, cancel, stream, sub.outCh, overlay, m.logger) unregister := func() { m.mu.Lock() @@ -409,7 +385,7 @@ func (m *GSOCEphemeralMode) runMux(stream p2p.Stream) { if msg == nil { continue } - m.subscriberConn.broadcast(msg) + m.subscriberConn.fanOut(msg) } } diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 6a8ccc90079..8020c4b3af6 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/ethersphere/bee/v2/pkg/bzz" "github.com/ethersphere/bee/v2/pkg/crypto" @@ -32,6 +33,12 @@ const ( // Mode constants ModeGSOCEphemeral ModeID = 1 + // Service-level broker message types. + MsgTypePing byte = 0x03 + + // streamPingInterval is how often the broker sends a keepalive ping to each subscriber. + streamPingInterval = 30 * time.Second + // Wire format sizes SpanSize = swarm.SpanSize // pubsub span: 8-byte little-endian uint64 (matches bee-js Span.LENGTH) MaxPayload = swarm.ChunkSize @@ -48,6 +55,43 @@ var ( ErrTopicMismatch = errors.New("pubsub: topic address mismatch") ) +// startBrokerWriter starts a goroutine that drains outCh to stream and sends +// keepalive pings on every streamPingInterval tick. +func startBrokerWriter(ctx context.Context, cancel context.CancelFunc, stream p2p.Stream, outCh <-chan []byte, overlay swarm.Address, logger log.Logger) { + go func() { + ticker := time.NewTicker(streamPingInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case msg := <-outCh: + if err := writeRaw(stream, msg); err != nil { + logger.Info("broker write to subscriber failed", "peer", overlay, "error", err) + cancel() + return + } + logger.Info("broker wrote to subscriber", "peer", overlay, "size", len(msg)) + case <-ticker.C: + if err := writeRaw(stream, []byte{MsgTypePing}); err != nil { + cancel() + return + } + } + } + }() +} + +// readServiceMessage handles broker wire messages that are common to all modes. +// Returns (handled, err). The caller should return (nil, err) when handled is true. +func readServiceMessage(typeBuf byte) (handled bool, err error) { + switch typeBuf { + case MsgTypePing: + return true, nil + } + return false, nil +} + func newMode(topicAddr [32]byte, modeID ModeID, logger log.Logger) (Mode, error) { switch modeID { case ModeGSOCEphemeral: @@ -305,7 +349,8 @@ func (sc *SubscriberConn) Unsubscribe(id uint64) { } } -func (sc *SubscriberConn) broadcast(msg []byte) { +// fanOut broadcasts a message to all registered WS session channels. +func (sc *SubscriberConn) fanOut(msg []byte) { sc.subsMu.Lock() defer sc.subsMu.Unlock() for _, ch := range sc.subs { From d0b43fe997d1d520ba51e716cca9a35739124735 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 17:26:19 +0200 Subject: [PATCH 39/51] docs: correction of size in the span --- openapi/Swarm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index f3652654250..0a43f492dd1 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -2550,8 +2550,8 @@ paths: or subscriber (read-only) depending on the presence of GSOC headers. **WebSocket protocol:** - - Inbound (client → node, publisher only): raw SOC payload `[sig:65B][span:4B][payload:N B]` - - Outbound (node → client): raw SOC payload `[sig:65B][span:4B][payload:N B]` + - Inbound (client → node, publisher only): raw SOC payload `[sig:65B][span:8B][payload:N B]` + - Outbound (node → client): raw SOC payload `[sig:65B][span:8B][payload:N B]` tags: - Pubsub parameters: From 699ef1ec112bcb39924a75d758776d3d3ac61075 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 17:28:16 +0200 Subject: [PATCH 40/51] fix: removing duplicated stream close, leave only cancel --- pkg/api/pubsub.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 69f00df0738..317204882bf 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -117,9 +117,6 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { // Upgrade() hijacks the connection before returning an error, // so do NOT write an HTTP response here. cancel() - if sc := mode.GetSubscriberConn(); sc != nil { - _ = sc.Stream.Close() - } logger.Info("websocket upgrade failed", "error", err) logger.Error(nil, "websocket upgrade failed") return From bdb7453a9c9e53e84200b74353718f1ba174d2de Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 17:30:39 +0200 Subject: [PATCH 41/51] refactor: log out dropping messages when ws buffer is full --- pkg/pubsub/mode.go | 7 ++++--- pkg/pubsub/pubsub.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index de013952754..951a306884a 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -121,7 +121,7 @@ func (m *GSOCEphemeralMode) validatePublisher(headers p2p.Headers) error { // First delivery to each subscriber includes a handshake with SOC identity; subsequent are data-only. func (m *GSOCEphemeralMode) formatBroadcast(sub *brokerSubscriber, rawMsg []byte) []byte { if !sub.handshakeHappened { - // Handshake: [1B type=0x01][32B SOC ID][20B owner][65B sig][4B span][NB payload] + // Handshake: [1B type=0x01][32B SOC ID][20B owner][65B sig][8B span][NB payload] msg := make([]byte, 1+IDSize+OwnerSize+len(rawMsg)) msg[0] = MsgTypeHandshake copy(msg[1:1+IDSize], m.gsocID) @@ -131,14 +131,14 @@ func (m *GSOCEphemeralMode) formatBroadcast(sub *brokerSubscriber, rawMsg []byte return msg } - // Data: [1B type=0x02][65B sig][4B span][NB payload] + // Data: [1B type=0x02][65B sig][8B span][NB payload] msg := make([]byte, 1+len(rawMsg)) msg[0] = MsgTypeData copy(msg[1:], rawMsg) return msg } -// ReadPublisherMessage reads [65B sig][4B span][NB payload (max 4KB)] from the stream, +// ReadPublisherMessage reads [65B sig][8B span][NB payload (max 4KB)] from the stream, // constructs and validates the SOC chunk and returns that. func (m *GSOCEphemeralMode) ReadPublisherMessage(stream p2p.Stream) ([]byte, error) { sig := make([]byte, SigSize) @@ -366,6 +366,7 @@ func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swar Overlay: overlay, refs: 1, subs: make(map[uint64]chan []byte), + logger: m.logger, } m.subscriberConn = sc go m.runMux(stream) diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 8020c4b3af6..40269b70e64 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -326,6 +326,7 @@ type SubscriberConn struct { subsMu sync.Mutex subs map[uint64]chan []byte nextID uint64 + logger log.Logger } // Subscribe registers a new WS session and returns its per-session message channel. @@ -357,6 +358,7 @@ func (sc *SubscriberConn) fanOut(msg []byte) { select { case ch <- msg: default: + sc.logger.Warning("pubsub: subscriber ws channel full, dropping message") } } } From 387337fae05114fd0d204fbbd5bc4b2659df9550 Mon Sep 17 00:00:00 2001 From: nugaon Date: Wed, 29 Apr 2026 18:52:42 +0200 Subject: [PATCH 42/51] fix: clear subscriberConn in runMux defer to prevent stale reference and msgType refactor --- pkg/pubsub/mode.go | 17 +++++++++++------ pkg/pubsub/pubsub.go | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 951a306884a..9237615e9f8 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -375,8 +375,14 @@ func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swar // runMux reads broker messages from the shared p2p stream and broadcasts each to all // registered WS sessions. It exits when the stream closes or returns an error. +// On exit it immediately clears m.subscriberConn so new Connect calls open a fresh stream. func (m *GSOCEphemeralMode) runMux(stream p2p.Stream) { - defer m.subscriberConn.closeAll() + defer func() { + m.subscriberConn.closeAll() + m.mu.Lock() + m.subscriberConn = nil + m.mu.Unlock() + }() for { msg, err := m.ReadBrokerMessage(stream) if err != nil { @@ -397,14 +403,13 @@ func (m *GSOCEphemeralMode) GetSubscriberConn() *SubscriberConn { return m.subscriberConn } -// RemoveSubscriberConn decrements the ref count for the connection. -// When the last WS session exits, it closes the stream, stopping the mux goroutine. +// RemoveSubscriberConn decrements the ref count for conn. +// When the last WS session exits it closes the stream, stopping the mux goroutine. +// If the mux already died and cleared m.subscriberConn, refs are still tracked on conn +// so the stream is closed exactly once when refs reach zero. func (m *GSOCEphemeralMode) RemoveSubscriberConn(conn *SubscriberConn) { m.mu.Lock() defer m.mu.Unlock() - if m.subscriberConn != conn { - return - } conn.refs-- if conn.refs <= 0 { m.subscriberConn = nil diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 40269b70e64..308bd84d55b 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -34,7 +34,8 @@ const ( ModeGSOCEphemeral ModeID = 1 // Service-level broker message types. - MsgTypePing byte = 0x03 + // 0x01 is reserved for ping across all modes; mode-specific types start at 0x02. + MsgTypePing byte = 0x01 // streamPingInterval is how often the broker sends a keepalive ping to each subscriber. streamPingInterval = 30 * time.Second From 4ac575cb60e81e0d74cafe786506ce6abd861ffb Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 30 Apr 2026 18:06:32 +0200 Subject: [PATCH 43/51] fix: msg type in mode --- pkg/pubsub/mode.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index 9237615e9f8..ca2ccc41413 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -52,9 +52,9 @@ type Mode interface { // --- GSOC Ephemeral Mode (mode 1) --- const ( - // Message types (Broker → Subscriber) - MsgTypeHandshake byte = 0x01 - MsgTypeData byte = 0x02 + // Mode-specific message types (Broker → Subscriber); 0x01 is reserved for service-level ping. + MsgTypeHandshake byte = 0x02 + MsgTypeData byte = 0x03 ) // GSOCEphemeralMode implements Mode for GSOC ephemeral messaging. @@ -121,7 +121,7 @@ func (m *GSOCEphemeralMode) validatePublisher(headers p2p.Headers) error { // First delivery to each subscriber includes a handshake with SOC identity; subsequent are data-only. func (m *GSOCEphemeralMode) formatBroadcast(sub *brokerSubscriber, rawMsg []byte) []byte { if !sub.handshakeHappened { - // Handshake: [1B type=0x01][32B SOC ID][20B owner][65B sig][8B span][NB payload] + // Handshake: [1B type=0x02][32B SOC ID][20B owner][65B sig][8B span][NB payload] msg := make([]byte, 1+IDSize+OwnerSize+len(rawMsg)) msg[0] = MsgTypeHandshake copy(msg[1:1+IDSize], m.gsocID) @@ -131,7 +131,7 @@ func (m *GSOCEphemeralMode) formatBroadcast(sub *brokerSubscriber, rawMsg []byte return msg } - // Data: [1B type=0x02][65B sig][8B span][NB payload] + // Data: [1B type=0x03][65B sig][8B span][NB payload] msg := make([]byte, 1+len(rawMsg)) msg[0] = MsgTypeData copy(msg[1:], rawMsg) From 9420ee1b3e672adee1ad4f88c4a233dd9c1c9d97 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 12 May 2026 16:15:17 +0200 Subject: [PATCH 44/51] fix: remove devmode --- pkg/api/pubsub.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 317204882bf..85918ea2288 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -89,12 +89,6 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { return } - if s.beeMode == DevMode { - logger.Warning("pubsub endpoint is disabled in dev mode") - jsonhttp.BadRequest(w, errUnsupportedDevNodeOperation) - return - } - // Connect to broker peer ctx, cancel := context.WithCancel(context.Background()) mode, err := s.pubsubSvc.Connect(ctx, underlay, topicAddr, pubsub.ModeGSOCEphemeral, connectOpts) From c503fd9ae1bdcae7da051544b2f73f3e4ba83ed8 Mon Sep 17 00:00:00 2001 From: nugaon Date: Tue, 12 May 2026 17:46:55 +0200 Subject: [PATCH 45/51] refactor: remove api header and use only query --- openapi/Swarm.yaml | 2 +- openapi/SwarmCommon.yaml | 16 ++++++++-------- pkg/api/api.go | 13 ++++--------- pkg/api/pubsub.go | 31 ++++++++++--------------------- 4 files changed, 23 insertions(+), 39 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 0a43f492dd1..6835cafbae0 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -2547,7 +2547,7 @@ paths: summary: Connect to a pubsub topic via WebSocket description: | Opens a WebSocket connection to a pubsub topic. The connection acts as either a publisher (read+write) - or subscriber (read-only) depending on the presence of GSOC headers. + or subscriber (read-only) depending on the presence of GSOC query params. **WebSocket protocol:** - Inbound (client → node, publisher only): raw SOC payload `[sig:65B][span:8B][payload:N B]` diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 8b816b0613e..86ddc9bcd51 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1334,28 +1334,28 @@ components: description: "ACT history Unix timestamp" SwarmPubsubPeer: - in: header - name: swarm-pubsub-peer + in: query + name: peer schema: type: string required: true description: "Multiaddress of the broker peer to connect to for pubsub" SwarmPubsubGsocEthAddress: - in: header - name: swarm-pubsub-gsoc-eth-address + in: query + name: gsoc-eth-address schema: $ref: "#/components/schemas/HexString" required: false - description: "GSOC owner Ethereum address (20 bytes, hex-encoded) for publisher role. Required together with swarm-pubsub-gsoc-topic to upgrade to publisher." + description: "GSOC owner Ethereum address (20 bytes, hex-encoded) for publisher role. Required together with gsoc-topic to upgrade to publisher." SwarmPubsubGsocTopic: - in: header - name: swarm-pubsub-gsoc-topic + in: query + name: gsoc-topic schema: $ref: "#/components/schemas/HexString" required: false - description: "GSOC topic identifier (hex) for publisher role. Required together with swarm-pubsub-gsoc-eth-address to upgrade to publisher." + description: "GSOC topic identifier (hex) for publisher role. Required together with gsoc-eth-address to upgrade to publisher." responses: "200": diff --git a/pkg/api/api.go b/pkg/api/api.go index 15a5563d1d5..a519eccbf08 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -94,14 +94,10 @@ const ( SwarmActTimestampHeader = "Swarm-Act-Timestamp" SwarmActPublisherHeader = "Swarm-Act-Publisher" SwarmActHistoryAddressHeader = "Swarm-Act-History-Address" - SwarmPubsubPeerHeader = "Swarm-Pubsub-Peer" - SwarmPubsubGsocEthAddressHeader = "Swarm-Pubsub-Gsoc-Eth-Address" - SwarmPubsubGsocTopicHeader = "Swarm-Pubsub-Gsoc-Topic" - - ImmutableHeader = "Immutable" - GasPriceHeader = "Gas-Price" - GasLimitHeader = "Gas-Limit" - ETagHeader = "ETag" + ImmutableHeader = "Immutable" + GasPriceHeader = "Gas-Price" + GasLimitHeader = "Gas-Limit" + ETagHeader = "ETag" AuthorizationHeader = "Authorization" AcceptEncodingHeader = "Accept-Encoding" @@ -590,7 +586,6 @@ func (s *Service) corsHandler(h http.Handler) http.Handler { SwarmRedundancyStrategyHeader, SwarmRedundancyFallbackModeHeader, SwarmChunkRetrievalTimeoutHeader, SwarmLookAheadBufferSizeHeader, SwarmFeedIndexHeader, SwarmFeedIndexNextHeader, SwarmSocSignatureHeader, SwarmOnlyRootChunk, GasPriceHeader, GasLimitHeader, ImmutableHeader, SwarmActHeader, SwarmActTimestampHeader, SwarmActPublisherHeader, SwarmActHistoryAddressHeader, - SwarmPubsubPeerHeader, SwarmPubsubGsocEthAddressHeader, SwarmPubsubGsocTopicHeader, } allowedHeadersStr := strings.Join(allowedHeaders, ", ") diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 85918ea2288..1482100c236 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -38,42 +38,31 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { copy(topicAddr[:], h.Sum(nil)) } - // Required: underlay multiaddr — accept from header or query param (browsers cannot set WS headers) - peerHeader := r.Header.Get(SwarmPubsubPeerHeader) - if peerHeader == "" { - peerHeader = r.URL.Query().Get("swarm-pubsub-peer") - } - if peerHeader == "" { - jsonhttp.BadRequest(w, "missing Swarm-Pubsub-Peer header") + peerMultiaddr := r.URL.Query().Get("peer") + if peerMultiaddr == "" { + jsonhttp.BadRequest(w, "missing peer query param") return } - underlay, err := ma.NewMultiaddr(peerHeader) + underlay, err := ma.NewMultiaddr(peerMultiaddr) if err != nil { - logger.Info("invalid peer multiaddr", "value", peerHeader, "error", err) - jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Peer header") + logger.Info("invalid peer multiaddr", "value", peerMultiaddr, "error", err) + jsonhttp.BadRequest(w, "invalid peer query param") return } - // Optional: GSOC fields for Publisher upgrade — accept from header or query param var connectOpts pubsub.ConnectOptions - gsocEthAddrHex := r.Header.Get(SwarmPubsubGsocEthAddressHeader) - if gsocEthAddrHex == "" { - gsocEthAddrHex = r.URL.Query().Get("swarm-pubsub-gsoc-eth-address") - } - gsocTopicHex := r.Header.Get(SwarmPubsubGsocTopicHeader) - if gsocTopicHex == "" { - gsocTopicHex = r.URL.Query().Get("swarm-pubsub-gsoc-topic") - } + gsocEthAddrHex := r.URL.Query().Get("gsoc-eth-address") + gsocTopicHex := r.URL.Query().Get("gsoc-topic") if gsocEthAddrHex != "" && gsocTopicHex != "" { gsocOwner, err := hex.DecodeString(gsocEthAddrHex) if err != nil { - jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Gsoc-Eth-Address header") + jsonhttp.BadRequest(w, "invalid gsoc-eth-address query param") return } gsocID, err := hex.DecodeString(gsocTopicHex) if err != nil { - jsonhttp.BadRequest(w, "invalid Swarm-Pubsub-Gsoc-Topic header") + jsonhttp.BadRequest(w, "invalid gsoc-topic query param") return } connectOpts.GsocOwner = gsocOwner From 6bfe79f4805041b71349fb732ffe7d48a50db37c Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 15 May 2026 14:45:30 +0200 Subject: [PATCH 46/51] refactor: remove pubsub max connections --- cmd/bee/cmd/cmd.go | 2 -- cmd/bee/cmd/start.go | 1 - openapi/Swarm.yaml | 2 +- pkg/node/node.go | 4 +--- pkg/p2p/libp2p/libp2p.go | 11 ++--------- pkg/pubsub/pubsub.go | 10 +--------- 6 files changed, 5 insertions(+), 25 deletions(-) diff --git a/cmd/bee/cmd/cmd.go b/cmd/bee/cmd/cmd.go index 4fe64f60e86..7ca4d44fa3d 100644 --- a/cmd/bee/cmd/cmd.go +++ b/cmd/bee/cmd/cmd.go @@ -89,7 +89,6 @@ const ( optionAutoTLSRegistrationEndpoint = "autotls-registration-endpoint" optionAutoTLSCAEndpoint = "autotls-ca-endpoint" optionNamePubsubBrokerMode = "pubsub-broker-mode" - optionNamePubsubMaxConnections = "pubsub-max-connections" // blockchain-rpc optionNameBlockchainRpcEndpoint = "blockchain-rpc-endpoint" @@ -337,7 +336,6 @@ func (c *command) setAllFlags(cmd *cobra.Command) { cmd.Flags().String(optionAutoTLSRegistrationEndpoint, p2pforge.DefaultForgeEndpoint, "autotls registration endpoint") cmd.Flags().String(optionAutoTLSCAEndpoint, p2pforge.DefaultCAEndpoint, "autotls certificate authority endpoint") cmd.Flags().Bool(optionNamePubsubBrokerMode, true, "enable pubsub broker mode") - cmd.Flags().Int(optionNamePubsubMaxConnections, 0, "max pubsub connections per topic (0 = unlimited)") } // preRun must be called from every command's PreRunE, after which c.logger is diff --git a/cmd/bee/cmd/start.go b/cmd/bee/cmd/start.go index 7a09256c3fe..1dc3f839956 100644 --- a/cmd/bee/cmd/start.go +++ b/cmd/bee/cmd/start.go @@ -333,7 +333,6 @@ func buildBeeNode(ctx context.Context, c *command, cmd *cobra.Command, logger lo WelcomeMessage: c.config.GetString(optionWelcomeMessage), WhitelistedWithdrawalAddress: c.config.GetStringSlice(optionNameWhitelistedWithdrawalAddress), PubsubBrokerMode: c.config.GetBool(optionNamePubsubBrokerMode), - PubsubMaxConnections: c.config.GetInt(optionNamePubsubMaxConnections), }) return b, err diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 6835cafbae0..2e78c0989cd 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -2569,7 +2569,7 @@ paths: schema: type: integer required: false - description: WebSocket ping period in seconds (default: 60) + description: "WebSocket ping period in seconds (default: 60)" responses: "101": description: WebSocket upgrade successful diff --git a/pkg/node/node.go b/pkg/node/node.go index df37a9bb883..4330ac1acd7 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -195,7 +195,6 @@ type Options struct { WelcomeMessage string WhitelistedWithdrawalAddress []string PubsubBrokerMode bool - PubsubMaxConnections int } const ( @@ -672,7 +671,6 @@ func NewBee( Nonce: nonce, ValidateOverlay: chainEnabled, Registry: registry, - PubsubReservedStreamSlots: o.PubsubMaxConnections, }) if err != nil { return nil, fmt.Errorf("p2p service: %w", err) @@ -745,7 +743,7 @@ func NewBee( return nil, fmt.Errorf("init batch service: %w", err) } - pubsubSvc := pubsub.New(p2ps, logger, o.PubsubBrokerMode, o.PubsubMaxConnections) + pubsubSvc := pubsub.New(p2ps, logger, o.PubsubBrokerMode) if err = p2ps.AddProtocol(pubsubSvc.Protocol()); err != nil { return nil, fmt.Errorf("pubsub protocol: %w", err) } diff --git a/pkg/p2p/libp2p/libp2p.go b/pkg/p2p/libp2p/libp2p.go index 2c4d672c8a9..802fbd0264d 100644 --- a/pkg/p2p/libp2p/libp2p.go +++ b/pkg/p2p/libp2p/libp2p.go @@ -149,7 +149,6 @@ type Options struct { HeadersRWTimeout time.Duration Registry *prometheus.Registry autoTLSCertManager autoTLSCertManager - PubsubReservedStreamSlots int } func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay swarm.Address, addr string, ab addressbook.Putter, storer storage.StateStorer, lightNodes *lightnode.Container, logger log.Logger, tracer *tracing.Tracer, o Options) (s *Service, returnErr error) { @@ -209,17 +208,11 @@ func New(ctx context.Context, signer beecrypto.Signer, networkID uint64, overlay return nil, err } - // Tweak certain settings - inboundLimit := rcmgr.LimitVal(IncomingStreamCountLimit - o.PubsubReservedStreamSlots) - if inboundLimit < 0 { - inboundLimit = 0 - } - cfg := rcmgr.PartialLimitConfig{ System: rcmgr.ResourceLimits{ - Streams: inboundLimit + OutgoingStreamCountLimit, + Streams: IncomingStreamCountLimit + OutgoingStreamCountLimit, StreamsOutbound: OutgoingStreamCountLimit, - StreamsInbound: inboundLimit, + StreamsInbound: IncomingStreamCountLimit, }, } diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 308bd84d55b..57d0146a473 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -50,7 +50,6 @@ const ( var ( ErrBrokerDisabled = errors.New("pubsub: broker mode is disabled") - ErrMaxConnections = errors.New("pubsub: max connections reached") ErrInvalidHandshake = errors.New("pubsub: handshake verification failed") ErrWrongHeaders = errors.New("pubsub: wrong required headers") ErrTopicMismatch = errors.New("pubsub: topic address mismatch") @@ -138,16 +137,14 @@ type Service struct { p2p P2P logger log.Logger brokerMode bool - maxConns int modes map[TopicModeKey]Mode // (topic, mode) -> mode instance } -func New(p2p P2P, logger log.Logger, brokerMode bool, maxConns int) *Service { +func New(p2p P2P, logger log.Logger, brokerMode bool) *Service { s := &Service{ p2p: p2p, logger: logger.WithName(loggerName).Register(), brokerMode: brokerMode, - maxConns: maxConns, modes: make(map[TopicModeKey]Mode), } return s @@ -269,11 +266,6 @@ func (s *Service) brokerHandler(ctx context.Context, peer p2p.Peer, stream p2p.S return err } - if s.maxConns > 0 && m.SubscriberCount() >= s.maxConns { - _ = stream.Reset() - return ErrMaxConnections - } - return m.HandleBroker(ctx, peer, stream, headers) } From dda323791d9981ac66420be60032151d6f59637b Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 15 May 2026 15:33:15 +0200 Subject: [PATCH 47/51] fix: ci linting --- pkg/api/pubsub.go | 4 ++-- pkg/pubsub/pubsub.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/api/pubsub.go b/pkg/api/pubsub.go index 1482100c236..bb0b3dd33de 100644 --- a/pkg/api/pubsub.go +++ b/pkg/api/pubsub.go @@ -71,7 +71,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { } headers := struct { - KeepAlive time.Duration `map:"Swarm-Keep-Alive"` + KeepAlive int `map:"Swarm-Keep-Alive"` }{} if response := s.mapStructure(r.Header, &headers); response != nil { response("invalid header params", logger, w) @@ -106,7 +106,7 @@ func (s *Service) pubsubWsHandler(w http.ResponseWriter, r *http.Request) { } logger.Info("websocket upgrade successful") - pingPeriod := headers.KeepAlive * time.Second + pingPeriod := time.Duration(headers.KeepAlive) * time.Second if pingPeriod == 0 { pingPeriod = time.Minute } diff --git a/pkg/pubsub/pubsub.go b/pkg/pubsub/pubsub.go index 57d0146a473..dea26128da3 100644 --- a/pkg/pubsub/pubsub.go +++ b/pkg/pubsub/pubsub.go @@ -209,7 +209,7 @@ func (s *Service) Topics() []TopicInfo { s.mu.RLock() defer s.mu.RUnlock() - var topics []TopicInfo + topics := make([]TopicInfo, 0, len(s.modes)) for key, m := range s.modes { info := TopicInfo{ From 1b032bcaac1abfe6974a360ef9d3f10b526b79fa Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 15 May 2026 15:58:36 +0200 Subject: [PATCH 48/51] refactor: move pubsub schemas from headers to components section --- openapi/Swarm.yaml | 2 +- openapi/SwarmCommon.yaml | 53 ++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 2e78c0989cd..cd9f80f616c 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -2548,7 +2548,7 @@ paths: description: | Opens a WebSocket connection to a pubsub topic. The connection acts as either a publisher (read+write) or subscriber (read-only) depending on the presence of GSOC query params. - + **WebSocket protocol:** - Inbound (client → node, publisher only): raw SOC payload `[sig:65B][span:8B][payload:N B]` - Outbound (node → client): raw SOC payload `[sig:65B][span:8B][payload:N B]` diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 86ddc9bcd51..1d50bf8f5ba 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -1052,6 +1052,33 @@ components: properties: transactionHash: $ref: "#/components/schemas/TransactionHash" + + PubsubTopicInfo: + type: object + properties: + topicAddress: + type: string + description: "Hex-encoded topic address" + mode: + type: integer + description: "Pubsub mode identifier" + role: + type: string + description: "Role of this node: 'broker' or 'subscriber'" + connections: + type: array + items: + type: string + description: "List of connected peer overlays" + + PubsubTopicListResponse: + type: object + properties: + topics: + type: array + items: + $ref: "#/components/schemas/PubsubTopicInfo" + headers: SwarmTag: description: "Tag UID" @@ -1095,32 +1122,6 @@ components: required: false description: "Indicates which feed version was resolved (v1 or v2)" - PubsubTopicInfo: - type: object - properties: - topicAddress: - type: string - description: "Hex-encoded topic address" - mode: - type: integer - description: "Pubsub mode identifier" - role: - type: string - description: "Role of this node: 'broker' or 'subscriber'" - connections: - type: array - items: - type: string - description: "List of connected peer overlays" - - PubsubTopicListResponse: - type: object - properties: - topics: - type: array - items: - $ref: "#/components/schemas/PubsubTopicInfo" - parameters: GasPriceParameter: in: header From cfa2fb12f6f21f87da8351db249673837e120f89 Mon Sep 17 00:00:00 2001 From: nugaon Date: Fri, 15 May 2026 19:39:44 +0200 Subject: [PATCH 49/51] fix: subscriberConn null set concurrency --- pkg/pubsub/mode.go | 10 +++++++--- pkg/topology/kademlia/kademlia.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/pubsub/mode.go b/pkg/pubsub/mode.go index ca2ccc41413..73831a6152d 100644 --- a/pkg/pubsub/mode.go +++ b/pkg/pubsub/mode.go @@ -377,10 +377,14 @@ func (m *GSOCEphemeralMode) CreateSubscriberConn(stream p2p.Stream, overlay swar // registered WS sessions. It exits when the stream closes or returns an error. // On exit it immediately clears m.subscriberConn so new Connect calls open a fresh stream. func (m *GSOCEphemeralMode) runMux(stream p2p.Stream) { + // Capture once; RemoveSubscriberConn may set m.subscriberConn=nil concurrently. + sc := m.subscriberConn defer func() { - m.subscriberConn.closeAll() + sc.closeAll() m.mu.Lock() - m.subscriberConn = nil + if m.subscriberConn == sc { + m.subscriberConn = nil + } m.mu.Unlock() }() for { @@ -392,7 +396,7 @@ func (m *GSOCEphemeralMode) runMux(stream p2p.Stream) { if msg == nil { continue } - m.subscriberConn.fanOut(msg) + sc.fanOut(msg) } } diff --git a/pkg/topology/kademlia/kademlia.go b/pkg/topology/kademlia/kademlia.go index 87919438ff4..78451a986c2 100644 --- a/pkg/topology/kademlia/kademlia.go +++ b/pkg/topology/kademlia/kademlia.go @@ -991,7 +991,7 @@ func (k *Kad) connect(ctx context.Context, peer swarm.Address, ma []ma.Multiaddr case errors.Is(err, p2p.ErrUnsupportedAddresses): return err case err != nil: - k.logger.Debug("could not connect to peer", "peer_address", peer, "error", err) + k.logger.Info("could not connect to peer", "peer_address", peer, "error", err) retryTime := time.Now().Add(k.opt.TimeToRetry) var e *p2p.ConnectionBackoffError From 2fe4c3239c839965c6e457a11a98646c466b84c5 Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 21 May 2026 17:53:54 +0200 Subject: [PATCH 50/51] test: init --- pkg/api/api_test.go | 3 + pkg/api/pubsub_test.go | 382 ++++++++++++++++++++++++++++++++++++++ pkg/pubsub/export_test.go | 40 ++++ pkg/pubsub/mode_1_test.go | 379 +++++++++++++++++++++++++++++++++++++ pkg/pubsub/pubsub_test.go | 180 ++++++++++++++++++ 5 files changed, 984 insertions(+) create mode 100644 pkg/api/pubsub_test.go create mode 100644 pkg/pubsub/export_test.go create mode 100644 pkg/pubsub/mode_1_test.go create mode 100644 pkg/pubsub/pubsub_test.go diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 4dd7aa1d3f7..4c46af1f3cd 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -42,6 +42,7 @@ import ( "github.com/ethersphere/bee/v2/pkg/postage/postagecontract" contractMock "github.com/ethersphere/bee/v2/pkg/postage/postagecontract/mock" "github.com/ethersphere/bee/v2/pkg/pss" + "github.com/ethersphere/bee/v2/pkg/pubsub" "github.com/ethersphere/bee/v2/pkg/pusher" "github.com/ethersphere/bee/v2/pkg/resolver" resolverMock "github.com/ethersphere/bee/v2/pkg/resolver/mock" @@ -135,6 +136,7 @@ type testServerOptions struct { FullAPIDisabled bool ChequebookDisabled bool SwapDisabled bool + PubsubService *pubsub.Service } func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket.Conn, string, *chanStorer) { @@ -205,6 +207,7 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket. Staking: o.StakingContract, NodeStatus: o.NodeStatus, PinIntegrity: o.PinIntegrity, + PubsubService: o.PubsubService, } // By default bee mode is set to full mode. diff --git a/pkg/api/pubsub_test.go b/pkg/api/pubsub_test.go new file mode 100644 index 00000000000..45e01bed862 --- /dev/null +++ b/pkg/api/pubsub_test.go @@ -0,0 +1,382 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package api_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/bzz" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/pubsub" + "github.com/ethersphere/bee/v2/pkg/spinlock" + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/gorilla/websocket" + ma "github.com/multiformats/go-multiaddr" +) + +func dialWs(t *testing.T, listener, topic string) *websocket.Conn { + t.Helper() + u := url.URL{ + Scheme: "ws", + Host: listener, + Path: "/pubsub/" + topic, + RawQuery: "peer=/ip4/127.0.0.1/tcp/9000", + } + conn, resp, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + if resp != nil { + t.Fatalf("websocket dial failed status=%d: %v", resp.StatusCode, err) + } + t.Fatalf("websocket dial failed: %v", err) + } + return conn +} + +// pipeStreamAPI is a bidirectional p2p.Stream backed by an io.Pipe pair. +type pipeStreamAPI struct { + once sync.Once + pr *io.PipeReader + pw *io.PipeWriter +} + +func newPipeStreamAPI() *pipeStreamAPI { + pr, pw := io.Pipe() + return &pipeStreamAPI{pr: pr, pw: pw} +} + +func (s *pipeStreamAPI) Read(p []byte) (int, error) { return s.pr.Read(p) } +func (s *pipeStreamAPI) Write(p []byte) (int, error) { return s.pw.Write(p) } +func (s *pipeStreamAPI) ResponseHeaders() p2p.Headers { return nil } +func (s *pipeStreamAPI) Headers() p2p.Headers { return nil } +func (s *pipeStreamAPI) FullClose() error { return s.Close() } +func (s *pipeStreamAPI) Close() error { + s.once.Do(func() { s.pr.Close(); s.pw.Close() }) + return nil +} +func (s *pipeStreamAPI) Reset() error { return s.Close() } + +// pubsubMockP2P implements pubsub.P2P for testing. +// Only ConnectAllowLight and NewStream do real work; all other methods are stubs. +type pubsubMockP2P struct { + connectAllowLight func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) + newStream func(ctx context.Context, address swarm.Address, h p2p.Headers, protocol, version, stream string) (p2p.Stream, error) +} + +func (m *pubsubMockP2P) ConnectAllowLight(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) { + return m.connectAllowLight(ctx, addrs) +} +func (m *pubsubMockP2P) NewStream(ctx context.Context, address swarm.Address, h p2p.Headers, protocol, version, stream string) (p2p.Stream, error) { + return m.newStream(ctx, address, h, protocol, version, stream) +} +func (m *pubsubMockP2P) AddProtocol(p2p.ProtocolSpec) error { return nil } +func (m *pubsubMockP2P) Connect(context.Context, []ma.Multiaddr) (*bzz.Address, error) { + return nil, nil +} +func (m *pubsubMockP2P) Disconnect(swarm.Address, string) error { return nil } +func (m *pubsubMockP2P) Blocklist(swarm.Address, time.Duration, string) error { return nil } +func (m *pubsubMockP2P) NetworkStatus() p2p.NetworkStatus { return p2p.NetworkStatusAvailable } +func (m *pubsubMockP2P) Peers() []p2p.Peer { return nil } +func (m *pubsubMockP2P) Blocklisted(swarm.Address) (bool, error) { return false, nil } +func (m *pubsubMockP2P) BlocklistedPeers() ([]p2p.BlockListedPeer, error) { return nil, nil } +func (m *pubsubMockP2P) Addresses() ([]ma.Multiaddr, error) { return nil, nil } +func (m *pubsubMockP2P) SetPickyNotifier(p2p.PickyNotifier) {} +func (m *pubsubMockP2P) Halt() {} + +// nopStream is a p2p.Stream that never returns data and ignores all writes. +// It blocks reads until closed, simulating a long-lived idle broker connection. +type nopStream struct { + once sync.Once + done chan struct{} +} + +func newNopStream() *nopStream { return &nopStream{done: make(chan struct{})} } + +func (s *nopStream) Read(p []byte) (int, error) { + <-s.done + return 0, io.EOF +} +func (s *nopStream) Write(p []byte) (int, error) { return len(p), nil } +func (s *nopStream) Close() error { s.once.Do(func() { close(s.done) }); return nil } +func (s *nopStream) ResponseHeaders() p2p.Headers { return nil } +func (s *nopStream) Headers() p2p.Headers { return nil } +func (s *nopStream) FullClose() error { return s.Close() } +func (s *nopStream) Reset() error { return s.Close() } + +func newPubsubService(t *testing.T) (*pubsub.Service, *nopStream) { + t.Helper() + ns := newNopStream() + t.Cleanup(func() { _ = ns.Close() }) + + mockP2P := &pubsubMockP2P{ + connectAllowLight: func(_ context.Context, _ []ma.Multiaddr) (*bzz.Address, error) { + return &bzz.Address{Overlay: swarm.NewAddress(make([]byte, 32))}, nil + }, + newStream: func(_ context.Context, _ swarm.Address, _ p2p.Headers, _, _, _ string) (p2p.Stream, error) { + return ns, nil + }, + } + return pubsub.New(mockP2P, log.Noop, false), ns +} + +// newPubsubServiceMultiStream returns a pubsub.Service whose mock P2P creates a +// fresh pipe stream for every NewStream call and collects them for test control. +func newPubsubServiceMultiStream(t *testing.T) (*pubsub.Service, func() []*pipeStreamAPI) { + t.Helper() + var mu sync.Mutex + var streams []*pipeStreamAPI + + mockP2P := &pubsubMockP2P{ + connectAllowLight: func(_ context.Context, _ []ma.Multiaddr) (*bzz.Address, error) { + return &bzz.Address{Overlay: swarm.NewAddress(make([]byte, 32))}, nil + }, + newStream: func(_ context.Context, _ swarm.Address, _ p2p.Headers, _, _, _ string) (p2p.Stream, error) { + ps := newPipeStreamAPI() + mu.Lock() + streams = append(streams, ps) + mu.Unlock() + return ps, nil + }, + } + svc := pubsub.New(mockP2P, log.Noop, false) + return svc, func() []*pipeStreamAPI { + mu.Lock() + defer mu.Unlock() + return streams + } +} + +func TestPubsubList_NilService(t *testing.T) { + t.Parallel() + + client, _, _, _ := newTestServer(t, testServerOptions{}) + + resp, err := client.Get("/pubsub/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404, got %d", resp.StatusCode) + } +} + +func TestPubsubList_Empty(t *testing.T) { + t.Parallel() + + svc := pubsub.New(nil, log.Noop, true) + client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) + + resp, err := client.Get("/pubsub/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } +} + +func TestPubsubWs_MissingPeer(t *testing.T) { + t.Parallel() + + svc := pubsub.New(nil, log.Noop, false) + client, _, listener, _ := newTestServer(t, testServerOptions{PubsubService: svc}) + _ = listener + + resp, err := client.Get("/pubsub/testtopic") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for missing peer, got %d", resp.StatusCode) + } +} + +func TestPubsubWs_InvalidMultiaddr(t *testing.T) { + t.Parallel() + + svc := pubsub.New(nil, log.Noop, false) + client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) + + resp, err := client.Get("/pubsub/testtopic?peer=notamultiaddr") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for invalid multiaddr, got %d", resp.StatusCode) + } +} + +func TestPubsubWs_InvalidGsocEthAddress(t *testing.T) { + t.Parallel() + + svc := pubsub.New(nil, log.Noop, false) + client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) + + resp, err := client.Get("/pubsub/testtopic?peer=/ip4/127.0.0.1/tcp/9000&gsoc-eth-address=ZZZZ&gsoc-topic=aabb") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for invalid gsoc-eth-address, got %d", resp.StatusCode) + } +} + +func TestPubsubWs_InvalidGsocTopic(t *testing.T) { + t.Parallel() + + svc := pubsub.New(nil, log.Noop, false) + client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) + + resp, err := client.Get("/pubsub/testtopic?peer=/ip4/127.0.0.1/tcp/9000&gsoc-eth-address=aabbccddeeff001122334455667788990011223344&gsoc-topic=ZZZZ") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for invalid gsoc-topic, got %d", resp.StatusCode) + } +} + +func TestPubsubWs_SubscriberConnect(t *testing.T) { + t.Parallel() + + svc, nopSt := newPubsubService(t) + _ = nopSt + + _, _, listener, _ := newTestServer(t, testServerOptions{ + PubsubService: svc, + Logger: log.Noop, + }) + + u := url.URL{ + Scheme: "ws", + Host: listener, + Path: "/pubsub/testtopic", + RawQuery: "peer=/ip4/127.0.0.1/tcp/9000", + } + + conn, resp, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + if resp != nil { + t.Fatalf("websocket dial failed with status %d: %v", resp.StatusCode, err) + } + t.Fatalf("websocket dial failed: %v", err) + } + defer conn.Close() + + // Connection should be alive; send a close frame and verify clean teardown. + err = conn.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + if err != nil { + t.Fatalf("write close: %v", err) + } + + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + for { + _, _, err := conn.ReadMessage() + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + break + } + if err != nil { + break + } + } +} + +func TestPubsubWs_TwoTopicsListAndMessages(t *testing.T) { + t.Parallel() + + svc, getStreams := newPubsubServiceMultiStream(t) + client, _, listener, _ := newTestServer(t, testServerOptions{ + PubsubService: svc, + Logger: log.Noop, + }) + + // Open two WS connections on different topics. + conn1 := dialWs(t, listener, "topic-alpha") + defer conn1.Close() + conn2 := dialWs(t, listener, "topic-beta") + defer conn2.Close() + + // Wait until both topics appear as active subscribers. + err := spinlock.Wait(3*time.Second, func() bool { + return len(svc.Topics()) == 2 + }) + if err != nil { + t.Fatalf("timed out waiting for 2 active topics, got %d", len(svc.Topics())) + } + + // Broker sends a ping (0x01) on each stream to exercise message exchange. + // A ping is the simplest valid broker message; runMux eats it and keeps running. + streams := getStreams() + if len(streams) != 2 { + t.Fatalf("expected 2 streams, got %d", len(streams)) + } + for _, st := range streams { + if _, err := st.pw.Write([]byte{pubsub.MsgTypePing}); err != nil { + t.Fatalf("write ping to stream: %v", err) + } + } + + // Query the list endpoint — both topics must be present. + resp, err := client.Get("/pubsub/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var body struct { + Topics []pubsub.TopicInfo `json:"topics"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(body.Topics) != 2 { + t.Fatalf("expected 2 topics, got %d", len(body.Topics)) + } + + // Close conn1 and verify the topic list shrinks to 1. + _ = conn1.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + conn1.Close() + + if err := spinlock.Wait(3*time.Second, func() bool { + return len(svc.Topics()) == 1 + }); err != nil { + t.Fatalf("timed out waiting for topic count to drop to 1, got %d", len(svc.Topics())) + } + + // Close conn2 and verify no topics remain. + _ = conn2.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + conn2.Close() + + if err := spinlock.Wait(3*time.Second, func() bool { + return len(svc.Topics()) == 0 + }); err != nil { + t.Fatalf("timed out waiting for topic count to drop to 0, got %d", len(svc.Topics())) + } +} diff --git a/pkg/pubsub/export_test.go b/pkg/pubsub/export_test.go new file mode 100644 index 00000000000..7eb91d37f7e --- /dev/null +++ b/pkg/pubsub/export_test.go @@ -0,0 +1,40 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pubsub + +import "github.com/ethersphere/bee/v2/pkg/log" + +func (m *GSOCEphemeralMode) FormatBroadcast(sub *brokerSubscriber, rawMsg []byte) []byte { + return m.formatBroadcast(sub, rawMsg) +} + +func (m *GSOCEphemeralMode) SetGsocParams(gsocOwner, gsocID []byte) { + m.setGsocParams(gsocOwner, gsocID) +} + +func (m *GSOCEphemeralMode) GsocOwner() []byte { + m.mu.RLock() + defer m.mu.RUnlock() + return m.gsocOwner +} + +func NewBrokerSubscriber() *brokerSubscriber { + return &brokerSubscriber{} +} + +func (b *brokerSubscriber) SetHandshakeDone() { + b.handshakeHappened = true +} + +func NewTestSubscriberConn(logger log.Logger) *SubscriberConn { + return &SubscriberConn{ + subs: make(map[uint64]chan []byte), + logger: logger, + } +} + +func (sc *SubscriberConn) FanOut(msg []byte) { + sc.fanOut(msg) +} diff --git a/pkg/pubsub/mode_1_test.go b/pkg/pubsub/mode_1_test.go new file mode 100644 index 00000000000..b7dd74b7d4f --- /dev/null +++ b/pkg/pubsub/mode_1_test.go @@ -0,0 +1,379 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Testing out Mode 1 (GSoC Ephemeral) implementation + +package pubsub_test + +import ( + "bytes" + "encoding/binary" + "io" + "testing" + + "github.com/ethersphere/bee/v2/pkg/cac" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/pubsub" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// readerStream is a p2p.Stream whose Read side is backed by a bytes.Reader. +// Write calls succeed but are discarded. Close/Reset are no-ops. +type readerStream struct { + io.Reader + headers p2p.Headers +} + +func (r *readerStream) Write(p []byte) (int, error) { return len(p), nil } +func (r *readerStream) Close() error { return nil } +func (r *readerStream) ResponseHeaders() p2p.Headers { return nil } +func (r *readerStream) Headers() p2p.Headers { return r.headers } +func (r *readerStream) FullClose() error { return nil } +func (r *readerStream) Reset() error { return nil } + +func newReaderStream(data []byte, headers p2p.Headers) *readerStream { + return &readerStream{Reader: bytes.NewReader(data), headers: headers} +} + +// socTestCtx holds key material for building valid SOC publisher messages. +type socTestCtx struct { + signer crypto.Signer + owner []byte + gsocID []byte + topicAddr [32]byte +} + +func newSocTestCtx(t *testing.T) *socTestCtx { + t.Helper() + + privKey, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(privKey) + ownerAddr, err := signer.EthereumAddress() + if err != nil { + t.Fatal(err) + } + + gsocID := make([]byte, swarm.HashSize) + + topicAddress, err := soc.CreateAddress(gsocID, ownerAddr.Bytes()) + if err != nil { + t.Fatal(err) + } + + var topicArr [32]byte + copy(topicArr[:], topicAddress.Bytes()) + + return &socTestCtx{ + signer: signer, + owner: ownerAddr.Bytes(), + gsocID: gsocID, + topicAddr: topicArr, + } +} + +// buildPublisherMsg returns a valid pubsub publisher wire message: sig(65B)+span(8B)+payload. +func buildPublisherMsg(t *testing.T, tc *socTestCtx, payload []byte) []byte { + t.Helper() + + spanBytes := make([]byte, pubsub.SpanSize) + binary.LittleEndian.PutUint64(spanBytes, uint64(len(payload))) + + cacData := append(spanBytes, payload...) + ch, err := cac.NewWithDataSpan(cacData) + if err != nil { + t.Fatal(err) + } + + h := swarm.NewHasher() + _, _ = h.Write(tc.gsocID) + _, _ = h.Write(ch.Address().Bytes()) + toSign := h.Sum(nil) + + sig, err := tc.signer.Sign(toSign) + if err != nil { + t.Fatal(err) + } + + var msg []byte + msg = append(msg, sig...) + msg = append(msg, spanBytes...) + msg = append(msg, payload...) + return msg +} + +func TestFormatBroadcast_Handshake(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + mode.SetGsocParams(tc.owner, tc.gsocID) + + rawMsg := []byte("sig_span_payload_placeholder_data_65_8_N") + sub := pubsub.NewBrokerSubscriber() + + out := mode.FormatBroadcast(sub, rawMsg) + + if out[0] != pubsub.MsgTypeHandshake { + t.Fatalf("expected type 0x%02x, got 0x%02x", pubsub.MsgTypeHandshake, out[0]) + } + + gotID := out[1 : 1+pubsub.IDSize] + if !bytes.Equal(gotID, tc.gsocID) { + t.Fatalf("gsocID mismatch: got %x want %x", gotID, tc.gsocID) + } + + gotOwner := out[1+pubsub.IDSize : 1+pubsub.IDSize+pubsub.OwnerSize] + if !bytes.Equal(gotOwner, tc.owner) { + t.Fatalf("owner mismatch: got %x want %x", gotOwner, tc.owner) + } + + gotPayload := out[1+pubsub.IDSize+pubsub.OwnerSize:] + if !bytes.Equal(gotPayload, rawMsg) { + t.Fatalf("payload mismatch: got %x want %x", gotPayload, rawMsg) + } +} + +func TestFormatBroadcast_HandshakeOnce(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + mode.SetGsocParams(tc.owner, tc.gsocID) + + rawMsg := []byte("data") + sub := pubsub.NewBrokerSubscriber() + + // First call: handshake + first := mode.FormatBroadcast(sub, rawMsg) + if first[0] != pubsub.MsgTypeHandshake { + t.Fatalf("expected handshake type on first call") + } + + // Second call: data only + second := mode.FormatBroadcast(sub, rawMsg) + if second[0] != pubsub.MsgTypeData { + t.Fatalf("expected data type on second call, got 0x%02x", second[0]) + } + if !bytes.Equal(second[1:], rawMsg) { + t.Fatalf("data payload mismatch") + } +} + +func TestFormatBroadcast_DataAfterHandshake(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + mode.SetGsocParams(tc.owner, tc.gsocID) + + rawMsg := []byte("payload") + sub := pubsub.NewBrokerSubscriber() + sub.SetHandshakeDone() + + out := mode.FormatBroadcast(sub, rawMsg) + + if out[0] != pubsub.MsgTypeData { + t.Fatalf("expected type 0x%02x, got 0x%02x", pubsub.MsgTypeData, out[0]) + } + if !bytes.Equal(out[1:], rawMsg) { + t.Fatalf("data payload mismatch") + } +} + +func TestReadBrokerMessage_Ping(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + + stream := newReaderStream([]byte{pubsub.MsgTypePing}, nil) + msg, err := mode.ReadBrokerMessage(stream) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if msg != nil { + t.Fatalf("expected nil msg for ping, got %v", msg) + } +} + +func TestReadBrokerMessage_Handshake(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + + payload := []byte("sziasztok!") + publisherFrame := buildPublisherMsg(t, tc, payload) + + // Assemble handshake message: [0x02][gsocID(32B)][owner(20B)][sig(65B)][span(8B)][payload] + var streamData []byte + streamData = append(streamData, pubsub.MsgTypeHandshake) + streamData = append(streamData, tc.gsocID...) + streamData = append(streamData, tc.owner...) + streamData = append(streamData, publisherFrame...) + + stream := newReaderStream(streamData, nil) + msg, err := mode.ReadBrokerMessage(stream) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if msg == nil { + t.Fatal("expected non-nil message from handshake") + } + + // Returned msg is [sig(65B)][span(8B)][payload]; verify it matches publisherFrame + if !bytes.Equal(msg, publisherFrame) { + t.Fatalf("returned message mismatch") + } +} + +func TestReadBrokerMessage_Data(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + mode.SetGsocParams(tc.owner, tc.gsocID) + + payload := []byte("data message") + publisherFrame := buildPublisherMsg(t, tc, payload) + + // Assemble data message: [0x03][sig(65B)][span(8B)][payload] + var streamData []byte + streamData = append(streamData, pubsub.MsgTypeData) + streamData = append(streamData, publisherFrame...) + + stream := newReaderStream(streamData, nil) + msg, err := mode.ReadBrokerMessage(stream) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !bytes.Equal(msg, publisherFrame) { + t.Fatalf("data message mismatch: got %x want %x", msg, publisherFrame) + } +} + +func TestReadBrokerMessage_InvalidSig(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + mode.SetGsocParams(tc.owner, tc.gsocID) + + payload := []byte("bad sig") + spanBytes := make([]byte, pubsub.SpanSize) + binary.LittleEndian.PutUint64(spanBytes, uint64(len(payload))) + badSig := make([]byte, pubsub.SigSize) // all zeros — invalid signature + + var streamData []byte + streamData = append(streamData, pubsub.MsgTypeData) + streamData = append(streamData, badSig...) + streamData = append(streamData, spanBytes...) + streamData = append(streamData, payload...) + + stream := newReaderStream(streamData, nil) + _, err := mode.ReadBrokerMessage(stream) + if err == nil { + t.Fatal("expected error for invalid signature, got nil") + } +} + +func TestSetGsocParams_AddressMismatch(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + + // Use a different ID that does not hash to topicAddr with tc.owner + wrongID := make([]byte, swarm.HashSize) + wrongID[0] = 0xff + + mode.SetGsocParams(tc.owner, wrongID) + + if mode.GsocOwner() != nil { + t.Fatal("expected gsocOwner to remain nil after address mismatch") + } +} + +func TestSetGsocParams_ValidParams(t *testing.T) { + t.Parallel() + + tc := newSocTestCtx(t) + mode := pubsub.NewGSOCEphemeralMode(tc.topicAddr[:], log.Noop) + + mode.SetGsocParams(tc.owner, tc.gsocID) + + if mode.GsocOwner() == nil { + t.Fatal("expected gsocOwner to be set after valid params") + } + if !bytes.Equal(mode.GsocOwner(), tc.owner) { + t.Fatalf("gsocOwner mismatch: got %x want %x", mode.GsocOwner(), tc.owner) + } +} + +func TestSubscriberConn_FanOut(t *testing.T) { + t.Parallel() + + sc := pubsub.NewTestSubscriberConn(log.Noop) + + id1, ch1 := sc.Subscribe() + id2, ch2 := sc.Subscribe() + defer sc.Unsubscribe(id1) + defer sc.Unsubscribe(id2) + + msg := []byte("broadcast") + sc.FanOut(msg) + + for i, ch := range []<-chan []byte{ch1, ch2} { + select { + case got := <-ch: + if !bytes.Equal(got, msg) { + t.Fatalf("subscriber %d: got %x want %x", i, got, msg) + } + default: + t.Fatalf("subscriber %d: no message received", i) + } + } +} + +func TestSubscriberConn_Unsubscribe(t *testing.T) { + t.Parallel() + + sc := pubsub.NewTestSubscriberConn(log.Noop) + + id, ch := sc.Subscribe() + sc.Unsubscribe(id) + + // Channel should be closed after unsubscribe. + select { + case _, ok := <-ch: + if ok { + t.Fatal("expected channel to be closed") + } + default: + t.Fatal("expected channel to be closed (not blocking)") + } +} + +func TestSubscriberConn_FanOut_SkipsFullChannel(t *testing.T) { + t.Parallel() + + sc := pubsub.NewTestSubscriberConn(log.Noop) + + id, ch := sc.Subscribe() + defer sc.Unsubscribe(id) + + // Fill the channel to capacity (16 slots). + for i := range cap(ch) { + sc.FanOut([]byte{byte(i)}) + } + + // This should not block even though the channel is full. + sc.FanOut([]byte("overflow")) +} diff --git a/pkg/pubsub/pubsub_test.go b/pkg/pubsub/pubsub_test.go new file mode 100644 index 00000000000..8de22d31ff5 --- /dev/null +++ b/pkg/pubsub/pubsub_test.go @@ -0,0 +1,180 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pubsub_test + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/pubsub" + "github.com/ethersphere/bee/v2/pkg/spinlock" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// pipeStream is a p2p.Stream backed by an io.Pipe pair for bidirectional use. +type pipeStream struct { + pr *io.PipeReader + pw *io.PipeWriter + headers p2p.Headers +} + +func (p *pipeStream) Read(b []byte) (int, error) { return p.pr.Read(b) } +func (p *pipeStream) Write(b []byte) (int, error) { return p.pw.Write(b) } +func (p *pipeStream) Close() error { p.pr.Close(); p.pw.Close(); return nil } +func (p *pipeStream) ResponseHeaders() p2p.Headers { return nil } +func (p *pipeStream) Headers() p2p.Headers { return p.headers } +func (p *pipeStream) FullClose() error { return p.Close() } +func (p *pipeStream) Reset() error { + p.pr.CloseWithError(io.ErrUnexpectedEOF) + p.pw.CloseWithError(io.ErrUnexpectedEOF) + return nil +} + +// newService creates a pubsub.Service with a nil P2P backend. +// This is valid for tests that never reach any P2P method call. +func newService(t *testing.T, brokerMode bool) *pubsub.Service { + t.Helper() + return pubsub.New(nil, log.Noop, brokerMode) +} + +func TestBrokerHandler_Disabled(t *testing.T) { + t.Parallel() + + svc := newService(t, false) + handler := svc.Protocol().StreamSpecs[0].Handler + + var topicAddr [32]byte + headers := p2p.Headers{ + pubsub.HeaderTopicAddress: topicAddr[:], + pubsub.HeaderMode: {byte(pubsub.ModeGSOCEphemeral)}, + pubsub.HeaderReadWrite: {0}, + } + stream := newReaderStream(nil, headers) + peer := p2p.Peer{Address: swarm.NewAddress(make([]byte, 32))} + + err := handler(context.Background(), peer, stream) + if !errors.Is(err, pubsub.ErrBrokerDisabled) { + t.Fatalf("expected ErrBrokerDisabled, got %v", err) + } +} + +func TestBrokerHandler_MissingTopicAddress(t *testing.T) { + t.Parallel() + + svc := newService(t, true) + handler := svc.Protocol().StreamSpecs[0].Handler + + headers := p2p.Headers{ + pubsub.HeaderMode: {byte(pubsub.ModeGSOCEphemeral)}, + } + stream := newReaderStream(nil, headers) + peer := p2p.Peer{Address: swarm.NewAddress(make([]byte, 32))} + + err := handler(context.Background(), peer, stream) + if !errors.Is(err, pubsub.ErrWrongHeaders) { + t.Fatalf("expected ErrWrongHeaders, got %v", err) + } +} + +func TestBrokerHandler_MissingMode(t *testing.T) { + t.Parallel() + + svc := newService(t, true) + handler := svc.Protocol().StreamSpecs[0].Handler + + var topicAddr [32]byte + headers := p2p.Headers{ + pubsub.HeaderTopicAddress: topicAddr[:], + } + stream := newReaderStream(nil, headers) + peer := p2p.Peer{Address: swarm.NewAddress(make([]byte, 32))} + + err := handler(context.Background(), peer, stream) + if !errors.Is(err, pubsub.ErrWrongHeaders) { + t.Fatalf("expected ErrWrongHeaders, got %v", err) + } +} + +func TestBrokerHandler_UnknownMode(t *testing.T) { + t.Parallel() + + svc := newService(t, true) + handler := svc.Protocol().StreamSpecs[0].Handler + + var topicAddr [32]byte + headers := p2p.Headers{ + pubsub.HeaderTopicAddress: topicAddr[:], + pubsub.HeaderMode: {0xff}, + } + stream := newReaderStream(nil, headers) + peer := p2p.Peer{Address: swarm.NewAddress(make([]byte, 32))} + + err := handler(context.Background(), peer, stream) + if err == nil { + t.Fatal("expected error for unknown mode, got nil") + } + if errors.Is(err, pubsub.ErrBrokerDisabled) || errors.Is(err, pubsub.ErrWrongHeaders) { + t.Fatalf("unexpected sentinel error for unknown mode: %v", err) + } +} + +func TestService_Topics_Empty(t *testing.T) { + t.Parallel() + + svc := newService(t, true) + if topics := svc.Topics(); len(topics) != 0 { + t.Fatalf("expected empty topics, got %d", len(topics)) + } +} + +func TestService_Topics_BrokerRole(t *testing.T) { + t.Parallel() + + svc := newService(t, true) + handler := svc.Protocol().StreamSpecs[0].Handler + + tc := newSocTestCtx(t) + headers := p2p.Headers{ + pubsub.HeaderTopicAddress: tc.topicAddr[:], + pubsub.HeaderMode: {byte(pubsub.ModeGSOCEphemeral)}, + pubsub.HeaderReadWrite: {0}, // subscriber on broker side + } + + pr, pw := io.Pipe() + stream := &pipeStream{pr: pr, pw: pw, headers: headers} + peer := p2p.Peer{Address: swarm.NewAddress(make([]byte, 32))} + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancel() + _ = stream.Close() + }) + + handlerErrCh := make(chan error, 1) + go func() { + handlerErrCh <- handler(ctx, peer, stream) + }() + + err := spinlock.Wait(time.Second, func() bool { + for _, topic := range svc.Topics() { + if topic.Role == "broker" { + return true + } + } + return false + }) + if err != nil { + t.Fatal("timed out waiting for broker topic to appear") + } + + cancel() + _ = stream.Close() + <-handlerErrCh +} From c3e2d2873988f6d41d25d36dc789b5fc7a41463a Mon Sep 17 00:00:00 2001 From: nugaon Date: Thu, 21 May 2026 18:42:07 +0200 Subject: [PATCH 51/51] fix: linting --- pkg/api/pubsub_test.go | 42 ++++++++++++++++++++++++++++++++------- pkg/pubsub/mode_1_test.go | 14 +++++++------ 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/pkg/api/pubsub_test.go b/pkg/api/pubsub_test.go index 45e01bed862..5c1600927f9 100644 --- a/pkg/api/pubsub_test.go +++ b/pkg/api/pubsub_test.go @@ -160,7 +160,11 @@ func TestPubsubList_NilService(t *testing.T) { client, _, _, _ := newTestServer(t, testServerOptions{}) - resp, err := client.Get("/pubsub/") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -177,7 +181,11 @@ func TestPubsubList_Empty(t *testing.T) { svc := pubsub.New(nil, log.Noop, true) client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) - resp, err := client.Get("/pubsub/") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -195,7 +203,11 @@ func TestPubsubWs_MissingPeer(t *testing.T) { client, _, listener, _ := newTestServer(t, testServerOptions{PubsubService: svc}) _ = listener - resp, err := client.Get("/pubsub/testtopic") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/testtopic", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -212,7 +224,11 @@ func TestPubsubWs_InvalidMultiaddr(t *testing.T) { svc := pubsub.New(nil, log.Noop, false) client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) - resp, err := client.Get("/pubsub/testtopic?peer=notamultiaddr") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/testtopic?peer=notamultiaddr", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -229,7 +245,11 @@ func TestPubsubWs_InvalidGsocEthAddress(t *testing.T) { svc := pubsub.New(nil, log.Noop, false) client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) - resp, err := client.Get("/pubsub/testtopic?peer=/ip4/127.0.0.1/tcp/9000&gsoc-eth-address=ZZZZ&gsoc-topic=aabb") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/testtopic?peer=/ip4/127.0.0.1/tcp/9000&gsoc-eth-address=ZZZZ&gsoc-topic=aabb", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -246,7 +266,11 @@ func TestPubsubWs_InvalidGsocTopic(t *testing.T) { svc := pubsub.New(nil, log.Noop, false) client, _, _, _ := newTestServer(t, testServerOptions{PubsubService: svc}) - resp, err := client.Get("/pubsub/testtopic?peer=/ip4/127.0.0.1/tcp/9000&gsoc-eth-address=aabbccddeeff001122334455667788990011223344&gsoc-topic=ZZZZ") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/testtopic?peer=/ip4/127.0.0.1/tcp/9000&gsoc-eth-address=aabbccddeeff001122334455667788990011223344&gsoc-topic=ZZZZ", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } @@ -339,7 +363,11 @@ func TestPubsubWs_TwoTopicsListAndMessages(t *testing.T) { } // Query the list endpoint — both topics must be present. - resp, err := client.Get("/pubsub/") + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/pubsub/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) if err != nil { t.Fatal(err) } diff --git a/pkg/pubsub/mode_1_test.go b/pkg/pubsub/mode_1_test.go index b7dd74b7d4f..d4ad201c332 100644 --- a/pkg/pubsub/mode_1_test.go +++ b/pkg/pubsub/mode_1_test.go @@ -85,7 +85,9 @@ func buildPublisherMsg(t *testing.T, tc *socTestCtx, payload []byte) []byte { spanBytes := make([]byte, pubsub.SpanSize) binary.LittleEndian.PutUint64(spanBytes, uint64(len(payload))) - cacData := append(spanBytes, payload...) + cacData := make([]byte, 0, pubsub.SpanSize+len(payload)) + cacData = append(cacData, spanBytes...) + cacData = append(cacData, payload...) ch, err := cac.NewWithDataSpan(cacData) if err != nil { t.Fatal(err) @@ -101,9 +103,9 @@ func buildPublisherMsg(t *testing.T, tc *socTestCtx, payload []byte) []byte { t.Fatal(err) } - var msg []byte + msg := make([]byte, 0, len(sig)+pubsub.SpanSize+len(payload)) msg = append(msg, sig...) - msg = append(msg, spanBytes...) + msg = append(msg, spanBytes[:pubsub.SpanSize]...) msg = append(msg, payload...) return msg } @@ -213,7 +215,7 @@ func TestReadBrokerMessage_Handshake(t *testing.T) { publisherFrame := buildPublisherMsg(t, tc, payload) // Assemble handshake message: [0x02][gsocID(32B)][owner(20B)][sig(65B)][span(8B)][payload] - var streamData []byte + streamData := make([]byte, 0, 1+len(tc.gsocID)+len(tc.owner)+len(publisherFrame)) streamData = append(streamData, pubsub.MsgTypeHandshake) streamData = append(streamData, tc.gsocID...) streamData = append(streamData, tc.owner...) @@ -245,7 +247,7 @@ func TestReadBrokerMessage_Data(t *testing.T) { publisherFrame := buildPublisherMsg(t, tc, payload) // Assemble data message: [0x03][sig(65B)][span(8B)][payload] - var streamData []byte + streamData := make([]byte, 0, 1+len(publisherFrame)) streamData = append(streamData, pubsub.MsgTypeData) streamData = append(streamData, publisherFrame...) @@ -271,7 +273,7 @@ func TestReadBrokerMessage_InvalidSig(t *testing.T) { binary.LittleEndian.PutUint64(spanBytes, uint64(len(payload))) badSig := make([]byte, pubsub.SigSize) // all zeros — invalid signature - var streamData []byte + streamData := make([]byte, 0, 1+len(badSig)+len(spanBytes)+len(payload)) streamData = append(streamData, pubsub.MsgTypeData) streamData = append(streamData, badSig...) streamData = append(streamData, spanBytes...)