From 44effd8f20a52c9d0e8fa9068bb4b42e35bdb6fe Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 23 Apr 2026 12:33:40 +0300 Subject: [PATCH 001/116] early exit on peer authentication messages, but keep broadcast them further --- integrationTests/testHeartbeatNode.go | 1 + .../metaInterceptorsContainerFactory.go | 1 + .../shardInterceptorsContainerFactory.go | 1 + .../interceptedPeerAuthentication.go | 12 +++++++ .../interceptedPeerAuthentication_test.go | 36 +++++++++++++++++++ .../factory/argInterceptedDataFactory.go | 1 + .../interceptedMetaHeaderDataFactory_test.go | 1 + ...nterceptedPeerAuthenticationDataFactory.go | 3 ++ .../peerAuthenticationInterceptorProcessor.go | 10 +++++- ...AuthenticationInterceptorProcessor_test.go | 35 ++++++++++++++++++ 10 files changed, 100 insertions(+), 1 deletion(-) diff --git a/integrationTests/testHeartbeatNode.go b/integrationTests/testHeartbeatNode.go index 23b13c40e3d..4bd94535633 100644 --- a/integrationTests/testHeartbeatNode.go +++ b/integrationTests/testHeartbeatNode.go @@ -645,6 +645,7 @@ func (thn *TestHeartbeatNode) initInterceptors() { SignaturesHandler: &processMock.SignaturesHandlerStub{}, HeartbeatExpiryTimespanInSec: thn.heartbeatExpiryTimespanInSec, PeerID: thn.MainMessenger.ID(), + PeerShardMapper: thn.MainPeerShardMapper, } thn.createPeerAuthInterceptor(argsFactory) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 8f6b8fc6b0a..1b561c28761 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -103,6 +103,7 @@ func NewMetaInterceptorsContainerFactory( SignaturesHandler: args.SignaturesHandler, HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, PeerID: args.MainMessenger.ID(), + PeerShardMapper: args.MainPeerShardMapper, } base := &baseInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index d144113d30f..7903041de61 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -104,6 +104,7 @@ func NewShardInterceptorsContainerFactory( SignaturesHandler: args.SignaturesHandler, HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, PeerID: args.MainMessenger.ID(), + PeerShardMapper: args.MainPeerShardMapper, } base := &baseInterceptorsContainerFactory{ diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index a10e5e6dd8d..8db1ca8f0cd 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -21,6 +21,7 @@ type ArgInterceptedPeerAuthentication struct { PeerSignatureHandler crypto.PeerSignatureHandler PayloadValidator process.PeerAuthenticationPayloadValidator HardforkTriggerPubKey []byte + PeerShardMapper process.PeerShardMapper } // interceptedPeerAuthentication is a wrapper over PeerAuthentication @@ -33,6 +34,7 @@ type interceptedPeerAuthentication struct { peerSignatureHandler crypto.PeerSignatureHandler payloadValidator process.PeerAuthenticationPayloadValidator hardforkTriggerPubKey []byte + peerShardMapper process.PeerShardMapper } // NewInterceptedPeerAuthentication tries to create a new intercepted peer authentication instance @@ -55,6 +57,7 @@ func NewInterceptedPeerAuthentication(arg ArgInterceptedPeerAuthentication) (*in peerSignatureHandler: arg.PeerSignatureHandler, payloadValidator: arg.PayloadValidator, hardforkTriggerPubKey: arg.HardforkTriggerPubKey, + peerShardMapper: arg.PeerShardMapper, } intercepted.peerId = core.PeerID(intercepted.peerAuthentication.Pid) @@ -81,6 +84,9 @@ func checkArg(arg ArgInterceptedPeerAuthentication) error { if len(arg.HardforkTriggerPubKey) == 0 { return fmt.Errorf("%w hardfork trigger public key bytes length is 0", process.ErrInvalidValue) } + if check.IfNil(arg.PeerShardMapper) { + return process.ErrNilPeerShardMapper + } return nil } @@ -135,6 +141,12 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { } } + // Early exit if mapping already exists + existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) + if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { + return nil + } + // Verify payload signature err = ipa.signaturesHandler.Verify(ipa.peerAuthentication.Payload, ipa.peerId, ipa.peerAuthentication.PayloadSignature) if err != nil { diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index fdc19e6a130..b16d2c85fbb 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var expectedErr = errors.New("expected error") @@ -59,6 +60,7 @@ func createMockInterceptedPeerAuthenticationArg(interceptedData *heartbeat.PeerA PeerSignatureHandler: &cryptoMocks.PeerSignatureHandlerStub{}, PayloadValidator: &testscommon.PeerAuthenticationPayloadValidatorStub{}, HardforkTriggerPubKey: providedHardforkPubKey, + PeerShardMapper: &processMocks.PeerShardMapperStub{}, } arg.DataBuff, _ = arg.Marshaller.Marshal(interceptedData) @@ -128,6 +130,16 @@ func TestNewInterceptedPeerAuthentication(t *testing.T) { assert.True(t, check.IfNil(ipa)) assert.Equal(t, process.ErrNilPeerSignatureHandler, err) }) + t.Run("nil peer shard mapper should error", func(t *testing.T) { + t.Parallel() + + arg := createMockInterceptedPeerAuthenticationArg(createDefaultInterceptedPeerAuthentication()) + arg.PeerShardMapper = nil + + ipa, err := NewInterceptedPeerAuthentication(arg) + assert.True(t, check.IfNil(ipa)) + assert.Equal(t, process.ErrNilPeerShardMapper, err) + }) t.Run("unmarshal returns error", func(t *testing.T) { t.Parallel() @@ -256,6 +268,30 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err = ipa.CheckValidity() assert.True(t, errors.Is(err, expectedErr)) }) + t.Run("peer already authenticated with same pubkey should early exit", func(t *testing.T) { + t.Parallel() + + providedPA := createDefaultInterceptedPeerAuthentication() + arg := createMockInterceptedPeerAuthenticationArg(providedPA) + + arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ + VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { + require.Fail(t, "should have not been called") + return expectedErr + }, + } + arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + return core.P2PPeerInfo{ + PkBytes: providedPA.Pubkey, + } + }, + } + + ipa, _ := NewInterceptedPeerAuthentication(arg) + err := ipa.CheckValidity() + assert.Nil(t, err) + }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/factory/argInterceptedDataFactory.go b/process/interceptors/factory/argInterceptedDataFactory.go index dbc7350436d..cb6d263e2b7 100644 --- a/process/interceptors/factory/argInterceptedDataFactory.go +++ b/process/interceptors/factory/argInterceptedDataFactory.go @@ -60,4 +60,5 @@ type ArgInterceptedDataFactory struct { SignaturesHandler process.SignaturesHandler HeartbeatExpiryTimespanInSec int64 PeerID core.PeerID + PeerShardMapper process.PeerShardMapper } diff --git a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go index f962fb9806e..6890990296b 100644 --- a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go +++ b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go @@ -138,6 +138,7 @@ func createMockArgument( SignaturesHandler: &processMocks.SignaturesHandlerStub{}, HeartbeatExpiryTimespanInSec: 30, PeerID: "pid", + PeerShardMapper: &processMocks.PeerShardMapperStub{}, } } diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index 18b4a4f40a2..a425dc3233a 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -21,6 +21,7 @@ type interceptedPeerAuthenticationDataFactory struct { peerSignatureHandler crypto.PeerSignatureHandler hardforkTriggerPubKey []byte payloadValidator process.PeerAuthenticationPayloadValidator + peerShardMapper process.PeerShardMapper } // NewInterceptedPeerAuthenticationDataFactory creates an instance of interceptedPeerAuthenticationDataFactory @@ -42,6 +43,7 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) peerSignatureHandler: arg.PeerSignatureHandler, payloadValidator: payloadValidator, hardforkTriggerPubKey: arg.CoreComponents.HardforkTriggerPubKey(), + peerShardMapper: arg.PeerShardMapper, }, nil } @@ -83,6 +85,7 @@ func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ cor PeerSignatureHandler: ipadf.peerSignatureHandler, PayloadValidator: ipadf.payloadValidator, HardforkTriggerPubKey: ipadf.hardforkTriggerPubKey, + PeerShardMapper: ipadf.peerShardMapper, } return heartbeat.NewInterceptedPeerAuthentication(arg) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 5864dcfcbf8..4633ee5aa7e 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -92,8 +92,16 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter } pidBytes := peerAuthenticationData.GetPid() + pid := core.PeerID(pidBytes) + + // early exit if info already saved + existingInfo := paip.peerShardMapper.GetPeerInfo(pid) + if string(existingInfo.PkBytes) == string(peerAuthenticationData.GetPubkey()) { + return nil + } + paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey()) + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) log.Trace("PeerAuthentication message saved") diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 3a1db0b6b66..2069c8d0d32 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" heartbeatMessages "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" @@ -63,6 +64,7 @@ func createMockInterceptedPeerAuthentication() process.InterceptedData { PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, PayloadValidator: payloadValidator, HardforkTriggerPubKey: []byte("provided hardfork pub key"), + PeerShardMapper: &mock.PeerShardMapperStub{}, } arg.DataBuff, _ = arg.Marshaller.Marshal(createInterceptedPeerAuthentication()) ipa, _ := heartbeat.NewInterceptedPeerAuthentication(arg) @@ -181,6 +183,39 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { err = paip.Save(createMockInterceptedPeerAuthentication(), "", "") assert.Equal(t, expectedError, err) }) + t.Run("peer info already saved should early exit", func(t *testing.T) { + t.Parallel() + + providedIPA := createMockInterceptedPeerAuthentication() + providedIPAHandler := providedIPA.(interceptedDataHandler) + providedIPAMessage := providedIPAHandler.Message().(*heartbeatMessages.PeerAuthentication) + + arg := createPeerAuthenticationInterceptorProcessArg() + arg.PeerAuthenticationCacher = &cache.CacherStub{ + PutCalled: func(key []byte, value interface{}, sizeInBytes int) (evicted bool) { + require.Fail(t, "should have not been called") + return false + }, + } + wasGetPeerInfoCalled := false + arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + wasGetPeerInfoCalled = true + assert.Equal(t, providedIPAMessage.Pid, pid.Bytes()) + return core.P2PPeerInfo{ + PkBytes: providedIPAMessage.Pubkey, + } + }, + } + + paip, err := processor.NewPeerAuthenticationInterceptorProcessor(arg) + assert.Nil(t, err) + assert.False(t, paip.IsInterfaceNil()) + + err = paip.Save(providedIPA, "", "") + assert.Nil(t, err) + assert.True(t, wasGetPeerInfoCalled) + }) t.Run("should work", func(t *testing.T) { t.Parallel() From bb16d6b109dc1e9d885b5f9fb76e1a3a476170d7 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 23 Apr 2026 13:01:44 +0300 Subject: [PATCH 002/116] save the messages even though map already exists --- .../peerAuthenticationInterceptorProcessor.go | 9 +++------ .../peerAuthenticationInterceptorProcessor_test.go | 14 +++++--------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 4633ee5aa7e..1718756a806 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -94,15 +94,12 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter pidBytes := peerAuthenticationData.GetPid() pid := core.PeerID(pidBytes) - // early exit if info already saved + paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) existingInfo := paip.peerShardMapper.GetPeerInfo(pid) - if string(existingInfo.PkBytes) == string(peerAuthenticationData.GetPubkey()) { - return nil + if string(existingInfo.PkBytes) != string(peerAuthenticationData.GetPubkey()) { + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) } - paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) - log.Trace("PeerAuthentication message saved") return nil diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 2069c8d0d32..1aa2da79b26 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -6,9 +6,6 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - heartbeatMessages "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/heartbeat" @@ -19,6 +16,8 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type interceptedDataHandler interface { @@ -191,12 +190,6 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { providedIPAMessage := providedIPAHandler.Message().(*heartbeatMessages.PeerAuthentication) arg := createPeerAuthenticationInterceptorProcessArg() - arg.PeerAuthenticationCacher = &cache.CacherStub{ - PutCalled: func(key []byte, value interface{}, sizeInBytes int) (evicted bool) { - require.Fail(t, "should have not been called") - return false - }, - } wasGetPeerInfoCalled := false arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { @@ -206,6 +199,9 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { PkBytes: providedIPAMessage.Pubkey, } }, + UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { + require.Fail(t, "should not have been called") + }, } paip, err := processor.NewPeerAuthenticationInterceptorProcessor(arg) From c9efcbd59baf5a7024ea273f448f492020941693 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 23 Apr 2026 15:10:06 +0300 Subject: [PATCH 003/116] return error when peer already authenticated --- process/errors.go | 3 ++ .../interceptedPeerAuthentication.go | 2 +- .../interceptedPeerAuthentication_test.go | 4 +-- .../peerAuthenticationInterceptorProcessor.go | 7 +---- ...AuthenticationInterceptorProcessor_test.go | 31 ------------------- 5 files changed, 7 insertions(+), 40 deletions(-) diff --git a/process/errors.go b/process/errors.go index ae4d7e9294f..2de790cb3f1 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1319,3 +1319,6 @@ var ErrInvalidChainParameters = errors.New("invalid chain parameters") // ErrDuplicatedHashInBlock signals that the same hash appears more than once where uniqueness is expected var ErrDuplicatedHashInBlock = errors.New("duplicated hash in block") + +// ErrPeerAlreadyAuthenticated signals that a peer authentication message was received for a peer that already has an existing mapping +var ErrPeerAlreadyAuthenticated = errors.New("peer already authenticated") diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 8db1ca8f0cd..9a205c56dda 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -144,7 +144,7 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { // Early exit if mapping already exists existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { - return nil + return process.ErrPeerAlreadyAuthenticated } // Verify payload signature diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index b16d2c85fbb..a48c3d4c7fd 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -268,7 +268,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err = ipa.CheckValidity() assert.True(t, errors.Is(err, expectedErr)) }) - t.Run("peer already authenticated with same pubkey should early exit", func(t *testing.T) { + t.Run("peer already authenticated with same pubkey should return error", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() @@ -290,7 +290,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { ipa, _ := NewInterceptedPeerAuthentication(arg) err := ipa.CheckValidity() - assert.Nil(t, err) + assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 1718756a806..5864dcfcbf8 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -92,13 +92,8 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter } pidBytes := peerAuthenticationData.GetPid() - pid := core.PeerID(pidBytes) - paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - existingInfo := paip.peerShardMapper.GetPeerInfo(pid) - if string(existingInfo.PkBytes) != string(peerAuthenticationData.GetPubkey()) { - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) - } + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey()) log.Trace("PeerAuthentication message saved") diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 1aa2da79b26..09016fbc0af 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) type interceptedDataHandler interface { @@ -182,36 +181,6 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { err = paip.Save(createMockInterceptedPeerAuthentication(), "", "") assert.Equal(t, expectedError, err) }) - t.Run("peer info already saved should early exit", func(t *testing.T) { - t.Parallel() - - providedIPA := createMockInterceptedPeerAuthentication() - providedIPAHandler := providedIPA.(interceptedDataHandler) - providedIPAMessage := providedIPAHandler.Message().(*heartbeatMessages.PeerAuthentication) - - arg := createPeerAuthenticationInterceptorProcessArg() - wasGetPeerInfoCalled := false - arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ - GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { - wasGetPeerInfoCalled = true - assert.Equal(t, providedIPAMessage.Pid, pid.Bytes()) - return core.P2PPeerInfo{ - PkBytes: providedIPAMessage.Pubkey, - } - }, - UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { - require.Fail(t, "should not have been called") - }, - } - - paip, err := processor.NewPeerAuthenticationInterceptorProcessor(arg) - assert.Nil(t, err) - assert.False(t, paip.IsInterfaceNil()) - - err = paip.Save(providedIPA, "", "") - assert.Nil(t, err) - assert.True(t, wasGetPeerInfoCalled) - }) t.Run("should work", func(t *testing.T) { t.Parallel() From 248ee7a85951514c45df446ab796b63d2759463e Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 24 Apr 2026 16:09:17 +0300 Subject: [PATCH 004/116] latest vm common from master --- factory/api/apiResolverFactory.go | 5 +++-- factory/processing/txSimulatorProcessComponents.go | 6 ++++-- go.mod | 4 ++-- go.sum | 8 ++++---- integrationTests/testProcessorNodeWithTestWebServer.go | 6 ++++-- integrationTests/vm/testInitializer.go | 5 +++-- node/external/transactionAPI/apiTransactionResults.go | 6 +++--- node/external/transactionAPI/interface.go | 2 +- node/external/transactionAPI/unmarshaller.go | 2 +- outport/process/transactionsfee/interface.go | 2 +- outport/process/transactionsfee/transactionChecker.go | 4 ++-- .../process/transactionsfee/transactionsFeeProcessor.go | 9 +++++---- process/transactionEvaluator/interface.go | 2 +- process/transactionEvaluator/transactionSimulator.go | 4 +++- testscommon/dataFieldParserStub.go | 2 +- 15 files changed, 38 insertions(+), 29 deletions(-) diff --git a/factory/api/apiResolverFactory.go b/factory/api/apiResolverFactory.go index feb1b1a4d24..e9f435853a6 100644 --- a/factory/api/apiResolverFactory.go +++ b/factory/api/apiResolverFactory.go @@ -222,8 +222,9 @@ func CreateApiResolver(args *ApiResolverArgs) (facade.ApiResolver, error) { } argsDataFieldParser := &datafield.ArgsOperationDataFieldParser{ - AddressLength: args.CoreComponents.AddressPubKeyConverter().Len(), - Marshalizer: args.CoreComponents.InternalMarshalizer(), + AddressLength: args.CoreComponents.AddressPubKeyConverter().Len(), + Marshalizer: args.CoreComponents.InternalMarshalizer(), + RelayedTransactionsV1V2DisableEpoch: args.CoreComponents.EnableEpochsHandler().GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), } dataFieldParser, err := datafield.NewOperationDataFieldParser(argsDataFieldParser) if err != nil { diff --git a/factory/processing/txSimulatorProcessComponents.go b/factory/processing/txSimulatorProcessComponents.go index 3b4878977e9..dcea258d65f 100644 --- a/factory/processing/txSimulatorProcessComponents.go +++ b/factory/processing/txSimulatorProcessComponents.go @@ -3,6 +3,7 @@ package processing import ( "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/disabled" bootstrapDisabled "github.com/multiversx/mx-chain-go/epochStart/bootstrap/disabled" "github.com/multiversx/mx-chain-go/factory" @@ -54,8 +55,9 @@ func (pcf *processComponentsFactory) createAPITransactionEvaluator(epochStartTri } dataFieldParser, err := datafield.NewOperationDataFieldParser(&datafield.ArgsOperationDataFieldParser{ - AddressLength: pcf.coreData.AddressPubKeyConverter().Len(), - Marshalizer: pcf.coreData.InternalMarshalizer(), + AddressLength: pcf.coreData.AddressPubKeyConverter().Len(), + Marshalizer: pcf.coreData.InternalMarshalizer(), + RelayedTransactionsV1V2DisableEpoch: pcf.coreData.EnableEpochsHandler().GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), }) if err != nil { return nil, nil, err diff --git a/go.mod b/go.mod index 3113541dea9..85e957d1889 100644 --- a/go.mod +++ b/go.mod @@ -19,11 +19,11 @@ require ( github.com/multiversx/mx-chain-communication-go v1.3.0 github.com/multiversx/mx-chain-core-go v1.4.1 github.com/multiversx/mx-chain-crypto-go v1.3.0 - github.com/multiversx/mx-chain-es-indexer-go v1.9.2 + github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 github.com/multiversx/mx-chain-storage-go v1.1.0 - github.com/multiversx/mx-chain-vm-common-go v1.6.5 + github.com/multiversx/mx-chain-vm-common-go v1.6.6 github.com/multiversx/mx-chain-vm-go v1.5.45 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 github.com/multiversx/mx-chain-vm-v1_3-go v1.3.70 diff --git a/go.sum b/go.sum index b1920975338..9caaf789bc5 100644 --- a/go.sum +++ b/go.sum @@ -405,16 +405,16 @@ github.com/multiversx/mx-chain-core-go v1.4.1 h1:ljs53jpdjtCohpaqm2n/dvTGrFlSgIp github.com/multiversx/mx-chain-core-go v1.4.1/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= -github.com/multiversx/mx-chain-es-indexer-go v1.9.2 h1:/K/cpTkwlFJ7zOD8VRhgc6ixi1t/3ua8CLl63LWHjvE= -github.com/multiversx/mx-chain-es-indexer-go v1.9.2/go.mod h1:t1rkD2vHXSI4EClig0h7+kRCSUCRrMF+emr4DHxFtfA= +github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= +github.com/multiversx/mx-chain-es-indexer-go v1.9.3/go.mod h1:dXRu2fmdiLFOcaRA34axQfoUcq8p9NUGqr4+9dN+p0Y= github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/WI0Qh112whgZNM= github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= -github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= -github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= +github.com/multiversx/mx-chain-vm-common-go v1.6.6 h1:BJSQndP8KSqcSIi47wQwQy3uBIn5rbT3213eJroVaog= +github.com/multiversx/mx-chain-vm-common-go v1.6.6/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= github.com/multiversx/mx-chain-vm-go v1.5.45/go.mod h1:Qc2Sckw+EfQwnapkzghFfhuUAOGv29oSZgvj8LJ+xWQ= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 h1:5gSR3IMw1mcp/v5oO+vZ5YOyWO8w7O2qKhCKNPwsWNE= diff --git a/integrationTests/testProcessorNodeWithTestWebServer.go b/integrationTests/testProcessorNodeWithTestWebServer.go index 792e43a5045..ec45aa10785 100644 --- a/integrationTests/testProcessorNodeWithTestWebServer.go +++ b/integrationTests/testProcessorNodeWithTestWebServer.go @@ -7,6 +7,7 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-vm-common-go/parsers" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" @@ -167,8 +168,9 @@ func createFacadeComponents(tpn *TestProcessorNode) nodeFacade.ApiResolver { log.LogIfError(err) argsDataFieldParser := &datafield.ArgsOperationDataFieldParser{ - AddressLength: TestAddressPubkeyConverter.Len(), - Marshalizer: TestMarshalizer, + AddressLength: TestAddressPubkeyConverter.Len(), + Marshalizer: TestMarshalizer, + RelayedTransactionsV1V2DisableEpoch: tpn.EnableEpochsHandler.GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), } dataFieldParser, err := datafield.NewOperationDataFieldParser(argsDataFieldParser) log.LogIfError(err) diff --git a/integrationTests/vm/testInitializer.go b/integrationTests/vm/testInitializer.go index 8e8944c6d6d..aadebd71402 100644 --- a/integrationTests/vm/testInitializer.go +++ b/integrationTests/vm/testInitializer.go @@ -995,8 +995,9 @@ func CreateTxProcessorWithOneSCExecutorWithVMs( }) dataFieldParser, err := datafield.NewOperationDataFieldParser(&datafield.ArgsOperationDataFieldParser{ - AddressLength: pubkeyConv.Len(), - Marshalizer: integrationtests.TestMarshalizer, + AddressLength: pubkeyConv.Len(), + Marshalizer: integrationtests.TestMarshalizer, + RelayedTransactionsV1V2DisableEpoch: enableEpochsHandler.GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), }) if err != nil { return nil, err diff --git a/node/external/transactionAPI/apiTransactionResults.go b/node/external/transactionAPI/apiTransactionResults.go index d4a89edfd15..a8a6af050f8 100644 --- a/node/external/transactionAPI/apiTransactionResults.go +++ b/node/external/transactionAPI/apiTransactionResults.go @@ -123,7 +123,7 @@ func (arp *apiTransactionResultsProcessor) getSmartContractResultsInTransactionB return nil, fmt.Errorf("%w: %v, hash = %s", errCannotLoadContractResults, err, hex.EncodeToString(scrHash)) } - scrAPI := arp.adaptSmartContractResult(scrHash, scr) + scrAPI := arp.adaptSmartContractResult(scrHash, scr, epoch) arp.loadLogsIntoContractResults(scrHash, epoch, scrAPI) @@ -171,7 +171,7 @@ func (arp *apiTransactionResultsProcessor) getScrFromStorage(hash []byte, epoch return scr, nil } -func (arp *apiTransactionResultsProcessor) adaptSmartContractResult(scrHash []byte, scr *smartContractResult.SmartContractResult) *transaction.ApiSmartContractResult { +func (arp *apiTransactionResultsProcessor) adaptSmartContractResult(scrHash []byte, scr *smartContractResult.SmartContractResult, epoch uint32) *transaction.ApiSmartContractResult { isRefund := arp.refundDetector.IsRefund(RefundDetectorInput{ Value: scr.Value.String(), Data: scr.Data, @@ -201,7 +201,7 @@ func (arp *apiTransactionResultsProcessor) adaptSmartContractResult(scrHash []by apiSCR.RelayerAddr, _ = arp.addressPubKeyConverter.Encode(scr.RelayerAddr) apiSCR.OriginalSender, _ = arp.addressPubKeyConverter.Encode(scr.OriginalSender) - res := arp.dataFieldParser.Parse(scr.Data, scr.GetSndAddr(), scr.GetRcvAddr(), arp.shardCoordinator.NumberOfShards()) + res := arp.dataFieldParser.Parse(scr.Data, scr.GetSndAddr(), scr.GetRcvAddr(), arp.shardCoordinator.NumberOfShards(), epoch) apiSCR.Operation = res.Operation apiSCR.Function = res.Function apiSCR.ESDTValues = res.ESDTValues diff --git a/node/external/transactionAPI/interface.go b/node/external/transactionAPI/interface.go index a32cac06184..cb64f15f9ab 100644 --- a/node/external/transactionAPI/interface.go +++ b/node/external/transactionAPI/interface.go @@ -28,5 +28,5 @@ type LogsFacade interface { // DataFieldParser defines what a data field parser should be able to do type DataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, epoch uint32) *datafield.ResponseParseData } diff --git a/node/external/transactionAPI/unmarshaller.go b/node/external/transactionAPI/unmarshaller.go index 42b2b21354c..ec5bc195c28 100644 --- a/node/external/transactionAPI/unmarshaller.go +++ b/node/external/transactionAPI/unmarshaller.go @@ -98,7 +98,7 @@ func (tu *txUnmarshaller) unmarshalTransaction( return nil, err } - res := tu.dataFieldParser.Parse(apiTx.Data, apiTx.Tx.GetSndAddr(), apiTx.Tx.GetRcvAddr(), tu.shardCoordinator.NumberOfShards()) + res := tu.dataFieldParser.Parse(apiTx.Data, apiTx.Tx.GetSndAddr(), apiTx.Tx.GetRcvAddr(), tu.shardCoordinator.NumberOfShards(), txEpoch) apiTx.Operation = res.Operation apiTx.Function = res.Function apiTx.ESDTValues = res.ESDTValues diff --git a/outport/process/transactionsfee/interface.go b/outport/process/transactionsfee/interface.go index 551ee59d1e2..b78f8e20741 100644 --- a/outport/process/transactionsfee/interface.go +++ b/outport/process/transactionsfee/interface.go @@ -23,5 +23,5 @@ type transactionGetter interface { } type dataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, epoch uint32) *datafield.ResponseParseData } diff --git a/outport/process/transactionsfee/transactionChecker.go b/outport/process/transactionsfee/transactionChecker.go index fd56d0c202b..830820c60f4 100644 --- a/outport/process/transactionsfee/transactionChecker.go +++ b/outport/process/transactionsfee/transactionChecker.go @@ -13,8 +13,8 @@ import ( vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) -func (tep *transactionsFeeProcessor) isESDTOperationWithSCCall(tx data.TransactionHandler) bool { - res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards()) +func (tep *transactionsFeeProcessor) isESDTOperationWithSCCall(tx data.TransactionHandler, epoch uint32) bool { + res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards(), epoch) isESDTTransferOperation := res.Operation == core.BuiltInFunctionESDTTransfer || res.Operation == core.BuiltInFunctionESDTNFTTransfer || res.Operation == core.BuiltInFunctionMultiESDTNFTTransfer diff --git a/outport/process/transactionsfee/transactionsFeeProcessor.go b/outport/process/transactionsfee/transactionsFeeProcessor.go index 728d625cfa6..de9f5d61c68 100644 --- a/outport/process/transactionsfee/transactionsFeeProcessor.go +++ b/outport/process/transactionsfee/transactionsFeeProcessor.go @@ -51,8 +51,9 @@ func NewTransactionsFeeProcessor(arg ArgTransactionsFeeProcessor) (*transactions } parser, err := datafield.NewOperationDataFieldParser(&datafield.ArgsOperationDataFieldParser{ - AddressLength: arg.PubKeyConverter.Len(), - Marshalizer: arg.Marshaller, + AddressLength: arg.PubKeyConverter.Len(), + Marshalizer: arg.Marshaller, + RelayedTransactionsV1V2DisableEpoch: arg.EnableEpochsHandler.GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), }) if err != nil { return nil, err @@ -131,7 +132,7 @@ func (tep *transactionsFeeProcessor) prepareNormalTxs(transactionsAndScrs *trans isRelayed := tep.isRelayedTxV1V2(txWithResult, epoch) isFeeFixActive := tep.enableEpochsHandler.IsFlagEnabledInEpoch(common.FixRelayedBaseCostFlag, epoch) isRelayedBeforeFix := isRelayed && !isFeeFixActive - if isRelayedBeforeFix || tep.isESDTOperationWithSCCall(txHandler) { + if isRelayedBeforeFix || tep.isESDTOperationWithSCCall(txHandler, epoch) { feeInfo.SetGasUsed(txWithResult.GetTxHandler().GetGasLimit()) feeInfo.SetFee(initialPaidFee) } @@ -259,7 +260,7 @@ func (tep *transactionsFeeProcessor) prepareTxWithResultsBasedOnLogs( return } - res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards()) + res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards(), epoch) if check.IfNilReflect(txWithResults.log) || (res.Function == "" && res.Operation == datafield.OperationTransfer) { return } diff --git a/process/transactionEvaluator/interface.go b/process/transactionEvaluator/interface.go index 0b6d2620d72..979e7098c61 100644 --- a/process/transactionEvaluator/interface.go +++ b/process/transactionEvaluator/interface.go @@ -15,5 +15,5 @@ type TransactionProcessor interface { // DataFieldParser defines what a data field parser should be able to do type DataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, epoch uint32) *datafield.ResponseParseData } diff --git a/process/transactionEvaluator/transactionSimulator.go b/process/transactionEvaluator/transactionSimulator.go index d01dc4ef85d..1fea49a2ac2 100644 --- a/process/transactionEvaluator/transactionSimulator.go +++ b/process/transactionEvaluator/transactionSimulator.go @@ -270,7 +270,9 @@ func (ts *transactionSimulator) adaptSmartContractResult(scr *smartContractResul ReturnMessage: string(scr.ReturnMessage), GasLimit: scr.GasLimit, }) - res := ts.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, ts.shardCoordinator.NumberOfShards()) + + currentEpoch := ts.blockChainHook.CurrentEpoch() + res := ts.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, ts.shardCoordinator.NumberOfShards(), currentEpoch) receiversEncoded, err := ts.addressPubKeyConverter.EncodeSlice(res.Receivers) if err != nil { diff --git a/testscommon/dataFieldParserStub.go b/testscommon/dataFieldParserStub.go index fcbe84497c7..f40a9117c59 100644 --- a/testscommon/dataFieldParserStub.go +++ b/testscommon/dataFieldParserStub.go @@ -8,7 +8,7 @@ type DataFieldParserStub struct { } // Parse - -func (df *DataFieldParserStub) Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData { +func (df *DataFieldParserStub) Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, _ uint32) *datafield.ResponseParseData { if df.ParseCalled != nil { return df.ParseCalled(dataField, sender, receiver, numOfShards) } From 0e6e94caca7be41db4506cbd28ab7c2983fff884 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 28 Apr 2026 15:46:16 +0300 Subject: [PATCH 005/116] add maxChunks check --- cmd/node/config/config.toml | 1 + config/config.go | 1 + .../epochStartInterceptorsContainerFactory.go | 1 + epochStart/bootstrap/process_test.go | 3 + factory/processing/processComponents.go | 2 + integrationTests/testConsensusNode.go | 1 + integrationTests/testFullNode.go | 1 + integrationTests/testProcessorNode.go | 2 + process/factory/interceptorscontainer/args.go | 1 + .../baseInterceptorsContainerFactory.go | 12 ++-- .../metaInterceptorsContainerFactory.go | 1 + .../metaInterceptorsContainerFactory_test.go | 1 + .../shardInterceptorsContainerFactory.go | 1 + .../shardInterceptorsContainerFactory_test.go | 1 + process/interceptors/processor/chunk/chunk.go | 5 ++ .../processor/trieNodeChunksProcessor.go | 30 ++++++++-- .../processor/trieNodeChunksProcessor_test.go | 55 +++++++++++++++++-- testscommon/generalConfig.go | 1 + 18 files changed, 106 insertions(+), 14 deletions(-) diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index ed93fbb82a2..ddb1d6acb7c 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -535,6 +535,7 @@ Enabled = true NumConcurrentResolverJobs = 50 NumConcurrentResolvingTrieNodesJobs = 3 + MaxAllowedTrieNodeChunks = 400 [Antiflood.FastReacting] IntervalInSeconds = 1 ReservedPercent = 20.0 diff --git a/config/config.go b/config/config.go index 0ad08a34111..7c75e72881d 100644 --- a/config/config.go +++ b/config/config.go @@ -398,6 +398,7 @@ type AntifloodConfig struct { Enabled bool NumConcurrentResolverJobs int32 NumConcurrentResolvingTrieNodesJobs int32 + MaxAllowedTrieNodeChunks uint32 OutOfSpecs FloodPreventerConfig FastReacting FloodPreventerConfig SlowReacting FloodPreventerConfig diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index 8700b1daa24..7fe30639999 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -106,6 +106,7 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), SignaturesHandler: args.SignaturesHandler, HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MaxAllowedTrieNodeChunks: args.Config.Antiflood.MaxAllowedTrieNodeChunks, MainPeerShardMapper: peerShardMapper, FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 6db1c836226..800f4db39bb 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -227,6 +227,9 @@ func createMockEpochStartBootstrapArgs( Shards: 10, }, Requesters: generalCfg.Requesters, + Antiflood: config.AntifloodConfig{ + MaxAllowedTrieNodeChunks: 400, + }, }, EconomicsData: &economicsmocks.EconomicsHandlerMock{ MinGasPriceCalled: func() uint64 { diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 23c1b01a051..4b854ae1efa 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1718,6 +1718,7 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), SignaturesHandler: pcf.network.NetworkMessenger(), HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, MainPeerShardMapper: mainPeerShardMapper, FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, @@ -1772,6 +1773,7 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), SignaturesHandler: pcf.network.NetworkMessenger(), HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, MainPeerShardMapper: mainPeerShardMapper, FullArchivePeerShardMapper: fullArchivePeerShardMapper, HardforkTrigger: hardforkTrigger, diff --git a/integrationTests/testConsensusNode.go b/integrationTests/testConsensusNode.go index 282d14b6bbd..6323de9f9e9 100644 --- a/integrationTests/testConsensusNode.go +++ b/integrationTests/testConsensusNode.go @@ -495,6 +495,7 @@ func (tcn *TestConsensusNode) initInterceptors( HardforkTrigger: &testscommon.HardforkTriggerStub{}, NodeOperationMode: common.NormalOperation, InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + MaxAllowedTrieNodeChunks: 400, } if tcn.ShardCoordinator.SelfId() == core.MetachainShardId { interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 4c122860f52..210377e60f6 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -748,6 +748,7 @@ func (tcn *TestFullNode) initInterceptors( HardforkTrigger: &testscommon.HardforkTriggerStub{}, NodeOperationMode: common.NormalOperation, InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + MaxAllowedTrieNodeChunks: 400, } if tcn.ShardCoordinator.SelfId() == core.MetachainShardId { interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 208c9183dab..3046e4d6914 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -1389,6 +1389,7 @@ func (tpn *TestProcessorNode) initInterceptors(heartbeatPk string) { HardforkTrigger: tpn.HardforkTrigger, NodeOperationMode: tpn.NodeOperationMode, InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + MaxAllowedTrieNodeChunks: 400, } interceptorContainerFactory, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorContainerFactoryArgs) @@ -1458,6 +1459,7 @@ func (tpn *TestProcessorNode) initInterceptors(heartbeatPk string) { HardforkTrigger: tpn.HardforkTrigger, NodeOperationMode: tpn.NodeOperationMode, InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + MaxAllowedTrieNodeChunks: 400, } interceptorContainerFactory, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(shardIntereptorContainerFactoryArgs) diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index 8e98c7c18ab..60b19fe75c5 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -40,6 +40,7 @@ type CommonInterceptorsContainerFactoryArgs struct { PeerSignatureHandler crypto.PeerSignatureHandler SignaturesHandler process.SignaturesHandler HeartbeatExpiryTimespanInSec int64 + MaxAllowedTrieNodeChunks uint32 MainPeerShardMapper process.PeerShardMapper FullArchivePeerShardMapper process.PeerShardMapper HardforkTrigger heartbeat.HardforkTrigger diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index bdd6ea118e1..9815378e7f7 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -51,6 +51,7 @@ type baseInterceptorsContainerFactory struct { preferredPeersHolder process.PreferredPeersHolderHandler hasher hashing.Hasher requestHandler process.RequestHandler + maxAllowedTrieNodeChunks uint32 mainPeerShardMapper process.PeerShardMapper fullArchivePeerShardMapper process.PeerShardMapper hardforkTrigger heartbeat.HardforkTrigger @@ -653,11 +654,12 @@ func (bicf *baseInterceptorsContainerFactory) createOneTrieNodesInterceptor(topi } argChunkProcessor := processor.TrieNodesChunksProcessorArgs{ - Hasher: bicf.hasher, - ChunksCacher: bicf.dataPool.TrieNodesChunks(), - RequestInterval: chunksProcessorRequestInterval, - RequestHandler: bicf.requestHandler, - Topic: topic, + Hasher: bicf.hasher, + ChunksCacher: bicf.dataPool.TrieNodesChunks(), + RequestInterval: chunksProcessorRequestInterval, + RequestHandler: bicf.requestHandler, + Topic: topic, + MaxAllowedChunks: bicf.maxAllowedTrieNodeChunks, } chunkProcessor, err := processor.NewTrieNodeChunksProcessor(argChunkProcessor) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 8f6b8fc6b0a..619b10b7357 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -124,6 +124,7 @@ func NewMetaInterceptorsContainerFactory( preferredPeersHolder: args.PreferredPeersHolder, hasher: args.CoreComponents.Hasher(), requestHandler: args.RequestHandler, + maxAllowedTrieNodeChunks: args.MaxAllowedTrieNodeChunks, mainPeerShardMapper: args.MainPeerShardMapper, fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, hardforkTrigger: args.HardforkTrigger, diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index eafb147747a..c740e185ca5 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -731,6 +731,7 @@ func getArgumentsMeta( PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, SignaturesHandler: &mock.SignaturesHandlerStub{}, HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 400, MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, HardforkTrigger: &testscommon.HardforkTriggerStub{}, diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index d144113d30f..5d3963c0e64 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -125,6 +125,7 @@ func NewShardInterceptorsContainerFactory( preferredPeersHolder: args.PreferredPeersHolder, hasher: args.CoreComponents.Hasher(), requestHandler: args.RequestHandler, + maxAllowedTrieNodeChunks: args.MaxAllowedTrieNodeChunks, mainPeerShardMapper: args.MainPeerShardMapper, fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, hardforkTrigger: args.HardforkTrigger, diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index b72d32ad037..5c9419eefda 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -763,6 +763,7 @@ func getArgumentsShard( PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, SignaturesHandler: &mock.SignaturesHandlerStub{}, HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 400, MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, HardforkTrigger: &testscommon.HardforkTriggerStub{}, diff --git a/process/interceptors/processor/chunk/chunk.go b/process/interceptors/processor/chunk/chunk.go index 44991c490ee..9edf585d88c 100644 --- a/process/interceptors/processor/chunk/chunk.go +++ b/process/interceptors/processor/chunk/chunk.go @@ -67,6 +67,11 @@ func (c *chunk) GetAllMissingChunkIndexes() []uint32 { return missing } +// MaxChunks returns the configured number of chunks for this chunked payload. +func (c *chunk) MaxChunks() uint32 { + return c.maxChunks +} + // Size returns the size in bytes stored in the values of the inner map func (c *chunk) Size() int { return c.size diff --git a/process/interceptors/processor/trieNodeChunksProcessor.go b/process/interceptors/processor/trieNodeChunksProcessor.go index f9c584562c4..475017796bc 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor.go +++ b/process/interceptors/processor/trieNodeChunksProcessor.go @@ -19,6 +19,7 @@ type chunkHandler interface { Put(chunkIndex uint32, buff []byte) TryAssembleAllChunks() []byte GetAllMissingChunkIndexes() []uint32 + MaxChunks() uint32 Size() int IsInterfaceNil() bool } @@ -30,11 +31,12 @@ type checkRequest struct { // TrieNodesChunksProcessorArgs is the argument DTO used in the trieNodeChunksProcessor constructor type TrieNodesChunksProcessorArgs struct { - Hasher hashing.Hasher - ChunksCacher storage.Cacher - RequestInterval time.Duration - RequestHandler process.RequestHandler - Topic string + Hasher hashing.Hasher + ChunksCacher storage.Cacher + RequestInterval time.Duration + RequestHandler process.RequestHandler + Topic string + MaxAllowedChunks uint32 } type trieNodeChunksProcessor struct { @@ -44,6 +46,7 @@ type trieNodeChunksProcessor struct { requestInterval time.Duration requestHandler process.RequestHandler topic string + maxAllowedChunks uint32 cancel func() chanClose chan struct{} } @@ -66,6 +69,9 @@ func NewTrieNodeChunksProcessor(arg TrieNodesChunksProcessorArgs) (*trieNodeChun if len(arg.Topic) == 0 { return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor", process.ErrEmptyTopic) } + if arg.MaxAllowedChunks < 2 { + return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor, MaxAllowedChunks should be at least 2", process.ErrInvalidValue) + } tncp := &trieNodeChunksProcessor{ hasher: arg.Hasher, @@ -74,6 +80,7 @@ func NewTrieNodeChunksProcessor(arg TrieNodesChunksProcessorArgs) (*trieNodeChun requestInterval: arg.RequestInterval, requestHandler: arg.RequestHandler, topic: arg.Topic, + maxAllowedChunks: arg.MaxAllowedChunks, chanClose: make(chan struct{}), } var ctx context.Context @@ -184,6 +191,10 @@ func (proc *trieNodeChunksProcessor) batchIsValid(b *batch.Batch, whiteListHandl if b.MaxChunks < 2 { return false, nil } + if b.MaxChunks > proc.maxAllowedChunks { + return false, fmt.Errorf("%w, trie node batch max chunks %d exceeds configured limit %d", + process.ErrInvalidValue, b.MaxChunks, proc.maxAllowedChunks) + } if len(b.Reference) != proc.hasher.Size() { return false, process.ErrIncompatibleReference } @@ -229,6 +240,15 @@ func (proc *trieNodeChunksProcessor) requestMissingForReference(reference []byte if !ok { return } + if chunkData.MaxChunks() > proc.maxAllowedChunks { + log.Warn("dropping cached trie node chunk tracker above configured limit", + "reference", reference, + "maxChunks", chunkData.MaxChunks(), + "configuredLimit", proc.maxAllowedChunks, + ) + proc.chunksCacher.Remove(reference) + return + } missing := chunkData.GetAllMissingChunkIndexes() for _, missingChunkIndex := range missing { diff --git a/process/interceptors/processor/trieNodeChunksProcessor_test.go b/process/interceptors/processor/trieNodeChunksProcessor_test.go index ad63ca7adc6..bdda540fd8b 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor_test.go +++ b/process/interceptors/processor/trieNodeChunksProcessor_test.go @@ -2,6 +2,7 @@ package processor import ( "bytes" + "context" "errors" "sync/atomic" "testing" @@ -11,6 +12,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/process/interceptors/processor/chunk" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cache" @@ -34,10 +36,11 @@ func createMockTrieNodesChunksProcessorArgs() TrieNodesChunksProcessorArgs { return 32 }, }, - ChunksCacher: cache.NewCacherMock(), - RequestInterval: time.Second, - RequestHandler: &testscommon.RequestHandlerStub{}, - Topic: "topic", + ChunksCacher: cache.NewCacherMock(), + RequestInterval: time.Second, + RequestHandler: &testscommon.RequestHandlerStub{}, + Topic: "topic", + MaxAllowedChunks: 3, } } @@ -91,6 +94,16 @@ func TestNewTrieNodeChunksProcessor_EmptyTopic(t *testing.T) { assert.True(t, check.IfNil(tncp)) } +func TestNewTrieNodeChunksProcessor_InvalidMaxAllowedChunks(t *testing.T) { + t.Parallel() + + args := createMockTrieNodesChunksProcessorArgs() + args.MaxAllowedChunks = 1 + tncp, err := NewTrieNodeChunksProcessor(args) + assert.True(t, errors.Is(err, process.ErrInvalidValue)) + assert.True(t, check.IfNil(tncp)) +} + func TestNewTrieNodeChunksProcessor_ShouldWork(t *testing.T) { t.Parallel() @@ -132,6 +145,18 @@ func TestTrieNodeChunksProcessor_CheckBatchInvalidBatch(t *testing.T) { assert.Equal(t, err, process.ErrIncompatibleReference) assert.Equal(t, emptyCheckedChunkResult, chunkResult) + chunkResult, err = tncp.CheckBatch( + &batch.Batch{ + Data: make([][]byte, 1), + Reference: make([]byte, 32), + ChunkIndex: 0, + MaxChunks: 4, + }, + createMockWhiteLister(true), + ) + assert.True(t, errors.Is(err, process.ErrInvalidValue)) + assert.Equal(t, emptyCheckedChunkResult, chunkResult) + chunkResult, err = tncp.CheckBatch( &batch.Batch{ Data: nil, @@ -276,6 +301,28 @@ func TestTrieNodeChunksProcessor_CheckBatchNotTheFirstBatch(t *testing.T) { assert.Equal(t, 1, args.ChunksCacher.Len()) } +func TestTrieNodeChunksProcessor_RequestMissingForReferenceShouldDropCachedChunkAboveConfiguredLimit(t *testing.T) { + t.Parallel() + + args := createMockTrieNodesChunksProcessorArgs() + numRequested := uint32(0) + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestTrieNodeCalled: func(_ []byte, _ string, _ uint32) { + atomic.AddUint32(&numRequested, 1) + }, + } + + tncp, _ := NewTrieNodeChunksProcessor(args) + args.ChunksCacher.Put(reference, chunk.NewChunk(args.MaxAllowedChunks+1, reference), 0) + + tncp.requestMissingForReference(reference, context.Background()) + + assert.Equal(t, 0, args.ChunksCacher.Len()) + assert.Equal(t, uint32(0), atomic.LoadUint32(&numRequested)) + + _ = tncp.Close() +} + func TestTrieNodeChunksProcessor_CheckBatchComponentClosed(t *testing.T) { t.Parallel() diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index 3448122e630..2b546238f0c 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -388,6 +388,7 @@ func GetGeneralConfig() config.Config { Antiflood: config.AntifloodConfig{ NumConcurrentResolverJobs: 2, NumConcurrentResolvingTrieNodesJobs: 1, + MaxAllowedTrieNodeChunks: 400, TxAccumulator: config.TxAccumulatorConfig{ MaxAllowedTimeInMilliseconds: 10, MaxDeviationTimeInMilliseconds: 1, From 0dac63f5c3ae63b03d6d4423f111bf44f53f089c Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 28 Apr 2026 17:40:05 +0300 Subject: [PATCH 006/116] add time limit for requesting trie chunks --- cmd/node/config/config.toml | 3 +- config/config.go | 23 ++-- .../epochStartInterceptorsContainerFactory.go | 65 ++++----- epochStart/bootstrap/process_test.go | 3 +- factory/processing/processComponents.go | 130 +++++++++--------- integrationTests/testConsensusNode.go | 65 ++++----- integrationTests/testFullNode.go | 65 ++++----- integrationTests/testProcessorNode.go | 130 +++++++++--------- process/factory/interceptorscontainer/args.go | 67 ++++----- .../baseInterceptorsContainerFactory.go | 66 ++++----- .../metaInterceptorsContainerFactory.go | 51 +++---- .../metaInterceptorsContainerFactory_test.go | 64 ++++----- .../shardInterceptorsContainerFactory.go | 51 +++---- .../shardInterceptorsContainerFactory_test.go | 64 ++++----- process/interceptors/processor/chunk/chunk.go | 24 +++- .../processor/trieNodeChunksProcessor.go | 62 +++++---- .../processor/trieNodeChunksProcessor_test.go | 47 ++++++- testscommon/generalConfig.go | 7 +- 18 files changed, 536 insertions(+), 451 deletions(-) diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index ddb1d6acb7c..c6dec26f7ec 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -535,7 +535,8 @@ Enabled = true NumConcurrentResolverJobs = 50 NumConcurrentResolvingTrieNodesJobs = 3 - MaxAllowedTrieNodeChunks = 400 + MaxAllowedTrieNodeChunks = 10 + TrieNodeChunksInactivityTimeoutInSec = 10 [Antiflood.FastReacting] IntervalInSeconds = 1 ReservedPercent = 20.0 diff --git a/config/config.go b/config/config.go index 7c75e72881d..748e2fe56a2 100644 --- a/config/config.go +++ b/config/config.go @@ -395,17 +395,18 @@ type TxAccumulatorConfig struct { // AntifloodConfig will hold all p2p antiflood parameters type AntifloodConfig struct { - Enabled bool - NumConcurrentResolverJobs int32 - NumConcurrentResolvingTrieNodesJobs int32 - MaxAllowedTrieNodeChunks uint32 - OutOfSpecs FloodPreventerConfig - FastReacting FloodPreventerConfig - SlowReacting FloodPreventerConfig - PeerMaxOutput AntifloodLimitsConfig - Cache CacheConfig - Topic TopicAntifloodConfig - TxAccumulator TxAccumulatorConfig + Enabled bool + NumConcurrentResolverJobs int32 + NumConcurrentResolvingTrieNodesJobs int32 + MaxAllowedTrieNodeChunks uint32 + TrieNodeChunksInactivityTimeoutInSec int64 + OutOfSpecs FloodPreventerConfig + FastReacting FloodPreventerConfig + SlowReacting FloodPreventerConfig + PeerMaxOutput AntifloodLimitsConfig + Cache CacheConfig + Topic TopicAntifloodConfig + TxAccumulator TxAccumulatorConfig } // FloodPreventerConfig will hold all flood preventer parameters diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index 7fe30639999..22d1db16d9c 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -80,38 +80,39 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) hardforkTrigger := disabledFactory.HardforkTrigger() containerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: args.CoreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: nodesCoordinator, - MainMessenger: args.MainMessenger, - FullArchiveMessenger: args.FullArchiveMessenger, - Store: storer, - DataPool: args.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: feeHandler, - BlockBlackList: blackListHandler, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: args.WhiteListHandler, - WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - AntifloodHandler: antiFloodHandler, - ArgumentsParser: args.ArgumentsParser, - PreferredPeersHolder: disabled.NewPreferredPeersHolder(), - SizeCheckDelta: uint32(sizeCheckDelta), - RequestHandler: args.RequestHandler, - PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), - SignaturesHandler: args.SignaturesHandler, - HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MaxAllowedTrieNodeChunks: args.Config.Antiflood.MaxAllowedTrieNodeChunks, - MainPeerShardMapper: peerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: args.NodeOperationMode, - InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + CoreComponents: args.CoreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: nodesCoordinator, + MainMessenger: args.MainMessenger, + FullArchiveMessenger: args.FullArchiveMessenger, + Store: storer, + DataPool: args.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: feeHandler, + BlockBlackList: blackListHandler, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: args.WhiteListHandler, + WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + AntifloodHandler: antiFloodHandler, + ArgumentsParser: args.ArgumentsParser, + PreferredPeersHolder: disabled.NewPreferredPeersHolder(), + SizeCheckDelta: uint32(sizeCheckDelta), + RequestHandler: args.RequestHandler, + PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), + SignaturesHandler: args.SignaturesHandler, + HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MaxAllowedTrieNodeChunks: args.Config.Antiflood.MaxAllowedTrieNodeChunks, + TrieNodeChunksInactivityTimeout: time.Duration(args.Config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, + MainPeerShardMapper: peerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: args.NodeOperationMode, + InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 800f4db39bb..239633fb312 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -228,7 +228,8 @@ func createMockEpochStartBootstrapArgs( }, Requesters: generalCfg.Requesters, Antiflood: config.AntifloodConfig{ - MaxAllowedTrieNodeChunks: 400, + TrieNodeChunksInactivityTimeoutInSec: 10, + MaxAllowedTrieNodeChunks: 10, }, }, EconomicsData: &economicsmocks.EconomicsHandlerMock{ diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 4b854ae1efa..a9e70324a97 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1692,38 +1692,39 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) shardInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - Accounts: pcf.state.AccountsAdapterAPI(), - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - NodesCoordinator: pcf.nodesCoordinator, - MainMessenger: pcf.network.NetworkMessenger(), - FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), - Store: pcf.data.StorageService(), - DataPool: pcf.data.Datapool(), - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: pcf.coreData.EconomicsData(), - BlockBlackList: headerBlackList, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: headerIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - AntifloodHandler: pcf.network.InputAntiFloodHandler(), - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), - SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, - RequestHandler: requestHandler, - PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), - SignaturesHandler: pcf.network.NetworkMessenger(), - HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, - MainPeerShardMapper: mainPeerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: nodeOperationMode, - InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + Accounts: pcf.state.AccountsAdapterAPI(), + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + NodesCoordinator: pcf.nodesCoordinator, + MainMessenger: pcf.network.NetworkMessenger(), + FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), + Store: pcf.data.StorageService(), + DataPool: pcf.data.Datapool(), + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: pcf.coreData.EconomicsData(), + BlockBlackList: headerBlackList, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: headerIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + AntifloodHandler: pcf.network.InputAntiFloodHandler(), + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), + SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, + RequestHandler: requestHandler, + PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), + SignaturesHandler: pcf.network.NetworkMessenger(), + HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, + TrieNodeChunksInactivityTimeout: time.Duration(pcf.config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, + MainPeerShardMapper: mainPeerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } interceptorContainerFactory, err := interceptorscontainer.NewShardInterceptorsContainerFactory(shardInterceptorsContainerFactoryArgs) @@ -1747,38 +1748,39 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) metaInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - NodesCoordinator: pcf.nodesCoordinator, - MainMessenger: pcf.network.NetworkMessenger(), - FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), - Store: pcf.data.StorageService(), - DataPool: pcf.data.Datapool(), - Accounts: pcf.state.AccountsAdapterAPI(), - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: pcf.coreData.EconomicsData(), - BlockBlackList: headerBlackList, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: headerIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - AntifloodHandler: pcf.network.InputAntiFloodHandler(), - ArgumentsParser: smartContract.NewArgumentParser(), - SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, - PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), - RequestHandler: requestHandler, - PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), - SignaturesHandler: pcf.network.NetworkMessenger(), - HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, - MainPeerShardMapper: mainPeerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: nodeOperationMode, - InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + NodesCoordinator: pcf.nodesCoordinator, + MainMessenger: pcf.network.NetworkMessenger(), + FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), + Store: pcf.data.StorageService(), + DataPool: pcf.data.Datapool(), + Accounts: pcf.state.AccountsAdapterAPI(), + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: pcf.coreData.EconomicsData(), + BlockBlackList: headerBlackList, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: headerIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + AntifloodHandler: pcf.network.InputAntiFloodHandler(), + ArgumentsParser: smartContract.NewArgumentParser(), + SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, + PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), + RequestHandler: requestHandler, + PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), + SignaturesHandler: pcf.network.NetworkMessenger(), + HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, + TrieNodeChunksInactivityTimeout: time.Duration(pcf.config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, + MainPeerShardMapper: mainPeerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorsContainerFactoryArgs) diff --git a/integrationTests/testConsensusNode.go b/integrationTests/testConsensusNode.go index 6323de9f9e9..d24ef3786c3 100644 --- a/integrationTests/testConsensusNode.go +++ b/integrationTests/testConsensusNode.go @@ -464,38 +464,39 @@ func (tcn *TestConsensusNode) initInterceptors( whiteListerVerifiedTxs, _ := interceptors.NewWhiteListDataVerifier(cacheVerified) interceptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: tcn.ShardCoordinator, - NodesCoordinator: tcn.NodesCoordinator, - MainMessenger: tcn.MainMessenger, - FullArchiveMessenger: tcn.FullArchiveMessenger, - Store: storage, - DataPool: tcn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: blockBlackListHandler, - HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), - ValidityAttester: blockTracker, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: whiteLstHandler, - WhiteListerVerifiedTxs: whiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), - FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), - MaxAllowedTrieNodeChunks: 400, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: tcn.ShardCoordinator, + NodesCoordinator: tcn.NodesCoordinator, + MainMessenger: tcn.MainMessenger, + FullArchiveMessenger: tcn.FullArchiveMessenger, + Store: storage, + DataPool: tcn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: blockBlackListHandler, + HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), + ValidityAttester: blockTracker, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: whiteLstHandler, + WhiteListerVerifiedTxs: whiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), + FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } if tcn.ShardCoordinator.SelfId() == core.MetachainShardId { interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 210377e60f6..fe08e9c9626 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -717,38 +717,39 @@ func (tcn *TestFullNode) initInterceptors( whiteListerVerifiedTxs, _ := interceptors.NewWhiteListDataVerifier(cacheVerified) interceptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: tcn.ShardCoordinator, - NodesCoordinator: tcn.NodesCoordinator, - MainMessenger: tcn.MainMessenger, - FullArchiveMessenger: tcn.FullArchiveMessenger, - Store: storage, - DataPool: tcn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: blockBlackListHandler, - HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), - ValidityAttester: blockTracker, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: whiteLstHandler, - WhiteListerVerifiedTxs: whiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), - FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), - MaxAllowedTrieNodeChunks: 400, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: tcn.ShardCoordinator, + NodesCoordinator: tcn.NodesCoordinator, + MainMessenger: tcn.MainMessenger, + FullArchiveMessenger: tcn.FullArchiveMessenger, + Store: storage, + DataPool: tcn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: blockBlackListHandler, + HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), + ValidityAttester: blockTracker, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: whiteLstHandler, + WhiteListerVerifiedTxs: whiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), + FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } if tcn.ShardCoordinator.SelfId() == core.MetachainShardId { interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 3046e4d6914..ae7facb9ba6 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -1358,38 +1358,39 @@ func (tpn *TestProcessorNode) initInterceptors(heartbeatPk string) { coreComponents.HardforkTriggerPubKeyField = providedHardforkPk metaInterceptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: tpn.AccntState, - ShardCoordinator: tpn.ShardCoordinator, - NodesCoordinator: tpn.NodesCoordinator, - MainMessenger: tpn.MainMessenger, - FullArchiveMessenger: tpn.FullArchiveMessenger, - Store: tpn.Storage, - DataPool: tpn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: tpn.EconomicsData, - BlockBlackList: tpn.BlockBlackListHandler, - HeaderSigVerifier: tpn.HeaderSigVerifier, - HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, - ValidityAttester: tpn.BlockTracker, - EpochStartTrigger: tpn.EpochStartTrigger, - WhiteListHandler: tpn.WhiteListHandler, - WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: tpn.RequestHandler, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MainPeerShardMapper: tpn.MainPeerShardMapper, - FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, - HardforkTrigger: tpn.HardforkTrigger, - NodeOperationMode: tpn.NodeOperationMode, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), - MaxAllowedTrieNodeChunks: 400, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: tpn.AccntState, + ShardCoordinator: tpn.ShardCoordinator, + NodesCoordinator: tpn.NodesCoordinator, + MainMessenger: tpn.MainMessenger, + FullArchiveMessenger: tpn.FullArchiveMessenger, + Store: tpn.Storage, + DataPool: tpn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: tpn.EconomicsData, + BlockBlackList: tpn.BlockBlackListHandler, + HeaderSigVerifier: tpn.HeaderSigVerifier, + HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, + ValidityAttester: tpn.BlockTracker, + EpochStartTrigger: tpn.EpochStartTrigger, + WhiteListHandler: tpn.WhiteListHandler, + WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: tpn.RequestHandler, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: tpn.MainPeerShardMapper, + FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, + HardforkTrigger: tpn.HardforkTrigger, + NodeOperationMode: tpn.NodeOperationMode, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } interceptorContainerFactory, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorContainerFactoryArgs) @@ -1428,38 +1429,39 @@ func (tpn *TestProcessorNode) initInterceptors(heartbeatPk string) { coreComponents.HardforkTriggerPubKeyField = providedHardforkPk shardIntereptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: tpn.AccntState, - ShardCoordinator: tpn.ShardCoordinator, - NodesCoordinator: tpn.NodesCoordinator, - MainMessenger: tpn.MainMessenger, - FullArchiveMessenger: tpn.FullArchiveMessenger, - Store: tpn.Storage, - DataPool: tpn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: tpn.EconomicsData, - BlockBlackList: tpn.BlockBlackListHandler, - HeaderSigVerifier: tpn.HeaderSigVerifier, - HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, - ValidityAttester: tpn.BlockTracker, - EpochStartTrigger: tpn.EpochStartTrigger, - WhiteListHandler: tpn.WhiteListHandler, - WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: tpn.RequestHandler, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MainPeerShardMapper: tpn.MainPeerShardMapper, - FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, - HardforkTrigger: tpn.HardforkTrigger, - NodeOperationMode: tpn.NodeOperationMode, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), - MaxAllowedTrieNodeChunks: 400, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: tpn.AccntState, + ShardCoordinator: tpn.ShardCoordinator, + NodesCoordinator: tpn.NodesCoordinator, + MainMessenger: tpn.MainMessenger, + FullArchiveMessenger: tpn.FullArchiveMessenger, + Store: tpn.Storage, + DataPool: tpn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: tpn.EconomicsData, + BlockBlackList: tpn.BlockBlackListHandler, + HeaderSigVerifier: tpn.HeaderSigVerifier, + HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, + ValidityAttester: tpn.BlockTracker, + EpochStartTrigger: tpn.EpochStartTrigger, + WhiteListHandler: tpn.WhiteListHandler, + WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: tpn.RequestHandler, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: tpn.MainPeerShardMapper, + FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, + HardforkTrigger: tpn.HardforkTrigger, + NodeOperationMode: tpn.NodeOperationMode, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } interceptorContainerFactory, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(shardIntereptorContainerFactoryArgs) diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index 60b19fe75c5..6a2832dd8ca 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -1,6 +1,8 @@ package interceptorscontainer import ( + "time" + crypto "github.com/multiversx/mx-chain-crypto-go" "github.com/multiversx/mx-chain-go/common" @@ -14,36 +16,37 @@ import ( // CommonInterceptorsContainerFactoryArgs holds the arguments needed for the metachain/shard interceptors factories type CommonInterceptorsContainerFactoryArgs struct { - CoreComponents process.CoreComponentsHolder - CryptoComponents process.CryptoComponentsHolder - Accounts state.AccountsAdapter - ShardCoordinator sharding.Coordinator - NodesCoordinator nodesCoordinator.NodesCoordinator - MainMessenger process.TopicHandler - FullArchiveMessenger process.TopicHandler - Store dataRetriever.StorageService - DataPool dataRetriever.PoolsHolder - MaxTxNonceDeltaAllowed int - TxFeeHandler process.FeeHandler - BlockBlackList process.TimeCacher - HeaderSigVerifier process.InterceptedHeaderSigVerifier - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - ValidityAttester process.ValidityAttester - EpochStartTrigger process.EpochStartTriggerHandler - WhiteListHandler process.WhiteListHandler - WhiteListerVerifiedTxs process.WhiteListHandler - AntifloodHandler process.P2PAntifloodHandler - ArgumentsParser process.ArgumentsParser - PreferredPeersHolder process.PreferredPeersHolderHandler - SizeCheckDelta uint32 - RequestHandler process.RequestHandler - PeerSignatureHandler crypto.PeerSignatureHandler - SignaturesHandler process.SignaturesHandler - HeartbeatExpiryTimespanInSec int64 - MaxAllowedTrieNodeChunks uint32 - MainPeerShardMapper process.PeerShardMapper - FullArchivePeerShardMapper process.PeerShardMapper - HardforkTrigger heartbeat.HardforkTrigger - NodeOperationMode common.NodeOperation - InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + CoreComponents process.CoreComponentsHolder + CryptoComponents process.CryptoComponentsHolder + Accounts state.AccountsAdapter + ShardCoordinator sharding.Coordinator + NodesCoordinator nodesCoordinator.NodesCoordinator + MainMessenger process.TopicHandler + FullArchiveMessenger process.TopicHandler + Store dataRetriever.StorageService + DataPool dataRetriever.PoolsHolder + MaxTxNonceDeltaAllowed int + TxFeeHandler process.FeeHandler + BlockBlackList process.TimeCacher + HeaderSigVerifier process.InterceptedHeaderSigVerifier + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + ValidityAttester process.ValidityAttester + EpochStartTrigger process.EpochStartTriggerHandler + WhiteListHandler process.WhiteListHandler + WhiteListerVerifiedTxs process.WhiteListHandler + AntifloodHandler process.P2PAntifloodHandler + ArgumentsParser process.ArgumentsParser + PreferredPeersHolder process.PreferredPeersHolderHandler + SizeCheckDelta uint32 + RequestHandler process.RequestHandler + PeerSignatureHandler crypto.PeerSignatureHandler + SignaturesHandler process.SignaturesHandler + HeartbeatExpiryTimespanInSec int64 + MaxAllowedTrieNodeChunks uint32 + TrieNodeChunksInactivityTimeout time.Duration + MainPeerShardMapper process.PeerShardMapper + FullArchivePeerShardMapper process.PeerShardMapper + HardforkTrigger heartbeat.HardforkTrigger + NodeOperationMode common.NodeOperation + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } diff --git a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go index 9815378e7f7..e8f5c887d0b 100644 --- a/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/baseInterceptorsContainerFactory.go @@ -32,32 +32,33 @@ const ( ) type baseInterceptorsContainerFactory struct { - mainContainer process.InterceptorsContainer - fullArchiveContainer process.InterceptorsContainer - shardCoordinator sharding.Coordinator - accounts state.AccountsAdapter - store dataRetriever.StorageService - dataPool dataRetriever.PoolsHolder - mainMessenger process.TopicHandler - fullArchiveMessenger process.TopicHandler - nodesCoordinator nodesCoordinator.NodesCoordinator - blockBlackList process.TimeCacher - argInterceptorFactory *interceptorFactory.ArgInterceptedDataFactory - globalThrottler process.InterceptorThrottler - maxTxNonceDeltaAllowed int - antifloodHandler process.P2PAntifloodHandler - whiteListHandler process.WhiteListHandler - whiteListerVerifiedTxs process.WhiteListHandler - preferredPeersHolder process.PreferredPeersHolderHandler - hasher hashing.Hasher - requestHandler process.RequestHandler - maxAllowedTrieNodeChunks uint32 - mainPeerShardMapper process.PeerShardMapper - fullArchivePeerShardMapper process.PeerShardMapper - hardforkTrigger heartbeat.HardforkTrigger - nodeOperationMode common.NodeOperation - interceptedDataVerifierFactory process.InterceptedDataVerifierFactory - enableEpochsHandler common.EnableEpochsHandler + mainContainer process.InterceptorsContainer + fullArchiveContainer process.InterceptorsContainer + shardCoordinator sharding.Coordinator + accounts state.AccountsAdapter + store dataRetriever.StorageService + dataPool dataRetriever.PoolsHolder + mainMessenger process.TopicHandler + fullArchiveMessenger process.TopicHandler + nodesCoordinator nodesCoordinator.NodesCoordinator + blockBlackList process.TimeCacher + argInterceptorFactory *interceptorFactory.ArgInterceptedDataFactory + globalThrottler process.InterceptorThrottler + maxTxNonceDeltaAllowed int + antifloodHandler process.P2PAntifloodHandler + whiteListHandler process.WhiteListHandler + whiteListerVerifiedTxs process.WhiteListHandler + preferredPeersHolder process.PreferredPeersHolderHandler + hasher hashing.Hasher + requestHandler process.RequestHandler + maxAllowedTrieNodeChunks uint32 + trieNodeChunksInactivityTimeout time.Duration + mainPeerShardMapper process.PeerShardMapper + fullArchivePeerShardMapper process.PeerShardMapper + hardforkTrigger heartbeat.HardforkTrigger + nodeOperationMode common.NodeOperation + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory + enableEpochsHandler common.EnableEpochsHandler } func checkBaseParams( @@ -654,12 +655,13 @@ func (bicf *baseInterceptorsContainerFactory) createOneTrieNodesInterceptor(topi } argChunkProcessor := processor.TrieNodesChunksProcessorArgs{ - Hasher: bicf.hasher, - ChunksCacher: bicf.dataPool.TrieNodesChunks(), - RequestInterval: chunksProcessorRequestInterval, - RequestHandler: bicf.requestHandler, - Topic: topic, - MaxAllowedChunks: bicf.maxAllowedTrieNodeChunks, + Hasher: bicf.hasher, + ChunksCacher: bicf.dataPool.TrieNodesChunks(), + RequestInterval: chunksProcessorRequestInterval, + RequestHandler: bicf.requestHandler, + Topic: topic, + MaxAllowedChunks: bicf.maxAllowedTrieNodeChunks, + ChunkInactivityTimeout: bicf.trieNodeChunksInactivityTimeout, } chunkProcessor, err := processor.NewTrieNodeChunksProcessor(argChunkProcessor) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 619b10b7357..48bd9f0577d 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -106,31 +106,32 @@ func NewMetaInterceptorsContainerFactory( } base := &baseInterceptorsContainerFactory{ - mainContainer: containers.NewInterceptorsContainer(), - fullArchiveContainer: containers.NewInterceptorsContainer(), - shardCoordinator: args.ShardCoordinator, - mainMessenger: args.MainMessenger, - fullArchiveMessenger: args.FullArchiveMessenger, - store: args.Store, - dataPool: args.DataPool, - nodesCoordinator: args.NodesCoordinator, - blockBlackList: args.BlockBlackList, - argInterceptorFactory: argInterceptorFactory, - maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, - accounts: args.Accounts, - antifloodHandler: args.AntifloodHandler, - whiteListHandler: args.WhiteListHandler, - whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - preferredPeersHolder: args.PreferredPeersHolder, - hasher: args.CoreComponents.Hasher(), - requestHandler: args.RequestHandler, - maxAllowedTrieNodeChunks: args.MaxAllowedTrieNodeChunks, - mainPeerShardMapper: args.MainPeerShardMapper, - fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, - hardforkTrigger: args.HardforkTrigger, - nodeOperationMode: args.NodeOperationMode, - interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, - enableEpochsHandler: args.CoreComponents.EnableEpochsHandler(), + mainContainer: containers.NewInterceptorsContainer(), + fullArchiveContainer: containers.NewInterceptorsContainer(), + shardCoordinator: args.ShardCoordinator, + mainMessenger: args.MainMessenger, + fullArchiveMessenger: args.FullArchiveMessenger, + store: args.Store, + dataPool: args.DataPool, + nodesCoordinator: args.NodesCoordinator, + blockBlackList: args.BlockBlackList, + argInterceptorFactory: argInterceptorFactory, + maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, + accounts: args.Accounts, + antifloodHandler: args.AntifloodHandler, + whiteListHandler: args.WhiteListHandler, + whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + preferredPeersHolder: args.PreferredPeersHolder, + hasher: args.CoreComponents.Hasher(), + requestHandler: args.RequestHandler, + maxAllowedTrieNodeChunks: args.MaxAllowedTrieNodeChunks, + trieNodeChunksInactivityTimeout: args.TrieNodeChunksInactivityTimeout, + mainPeerShardMapper: args.MainPeerShardMapper, + fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, + hardforkTrigger: args.HardforkTrigger, + nodeOperationMode: args.NodeOperationMode, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + enableEpochsHandler: args.CoreComponents.EnableEpochsHandler(), } icf := &metaInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index c740e185ca5..1d40dc9e81f 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -4,6 +4,7 @@ import ( "errors" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -706,36 +707,37 @@ func getArgumentsMeta( cryptoComp *mock.CryptoComponentsMock, ) interceptorscontainer.CommonInterceptorsContainerFactoryArgs { return interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComp, - CryptoComponents: cryptoComp, - Accounts: &stateMock.AccountsStub{}, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - MainMessenger: &mock.TopicHandlerStub{}, - FullArchiveMessenger: &mock.TopicHandlerStub{}, - Store: createMetaStore(), - DataPool: createMetaDataPools(), - MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: &testscommon.TimeCacheStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - ValidityAttester: &mock.ValidityAttesterStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - WhiteListHandler: &testscommon.WhiteListHandlerStub{}, - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - ArgumentsParser: &testscommon.ArgumentParserMock{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 400, - MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, + CoreComponents: coreComp, + CryptoComponents: cryptoComp, + Accounts: &stateMock.AccountsStub{}, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + MainMessenger: &mock.TopicHandlerStub{}, + FullArchiveMessenger: &mock.TopicHandlerStub{}, + Store: createMetaStore(), + DataPool: createMetaDataPools(), + MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: &testscommon.TimeCacheStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + ValidityAttester: &mock.ValidityAttesterStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + ArgumentsParser: &testscommon.ArgumentParserMock{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index 5d3963c0e64..541c65c9fa7 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -107,31 +107,32 @@ func NewShardInterceptorsContainerFactory( } base := &baseInterceptorsContainerFactory{ - mainContainer: containers.NewInterceptorsContainer(), - fullArchiveContainer: containers.NewInterceptorsContainer(), - accounts: args.Accounts, - shardCoordinator: args.ShardCoordinator, - mainMessenger: args.MainMessenger, - fullArchiveMessenger: args.FullArchiveMessenger, - store: args.Store, - dataPool: args.DataPool, - nodesCoordinator: args.NodesCoordinator, - argInterceptorFactory: argInterceptorFactory, - blockBlackList: args.BlockBlackList, - maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, - antifloodHandler: args.AntifloodHandler, - whiteListHandler: args.WhiteListHandler, - whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - preferredPeersHolder: args.PreferredPeersHolder, - hasher: args.CoreComponents.Hasher(), - requestHandler: args.RequestHandler, - maxAllowedTrieNodeChunks: args.MaxAllowedTrieNodeChunks, - mainPeerShardMapper: args.MainPeerShardMapper, - fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, - hardforkTrigger: args.HardforkTrigger, - nodeOperationMode: args.NodeOperationMode, - interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, - enableEpochsHandler: args.CoreComponents.EnableEpochsHandler(), + mainContainer: containers.NewInterceptorsContainer(), + fullArchiveContainer: containers.NewInterceptorsContainer(), + accounts: args.Accounts, + shardCoordinator: args.ShardCoordinator, + mainMessenger: args.MainMessenger, + fullArchiveMessenger: args.FullArchiveMessenger, + store: args.Store, + dataPool: args.DataPool, + nodesCoordinator: args.NodesCoordinator, + argInterceptorFactory: argInterceptorFactory, + blockBlackList: args.BlockBlackList, + maxTxNonceDeltaAllowed: args.MaxTxNonceDeltaAllowed, + antifloodHandler: args.AntifloodHandler, + whiteListHandler: args.WhiteListHandler, + whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + preferredPeersHolder: args.PreferredPeersHolder, + hasher: args.CoreComponents.Hasher(), + requestHandler: args.RequestHandler, + maxAllowedTrieNodeChunks: args.MaxAllowedTrieNodeChunks, + trieNodeChunksInactivityTimeout: args.TrieNodeChunksInactivityTimeout, + mainPeerShardMapper: args.MainPeerShardMapper, + fullArchivePeerShardMapper: args.FullArchivePeerShardMapper, + hardforkTrigger: args.HardforkTrigger, + nodeOperationMode: args.NodeOperationMode, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + enableEpochsHandler: args.CoreComponents.EnableEpochsHandler(), } icf := &shardInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index 5c9419eefda..8d4520beff7 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -4,6 +4,7 @@ import ( "errors" "strings" "testing" + "time" "github.com/multiversx/mx-chain-core-go/core/versioning" "github.com/stretchr/testify/assert" @@ -737,36 +738,37 @@ func getArgumentsShard( cryptoComp *mock.CryptoComponentsMock, ) interceptorscontainer.CommonInterceptorsContainerFactoryArgs { return interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComp, - CryptoComponents: cryptoComp, - Accounts: &stateMock.AccountsStub{}, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - MainMessenger: &mock.TopicHandlerStub{}, - FullArchiveMessenger: &mock.TopicHandlerStub{}, - Store: createShardStore(), - DataPool: createShardDataPools(), - MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: &testscommon.TimeCacheStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - SizeCheckDelta: 0, - ValidityAttester: &mock.ValidityAttesterStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListHandler: &testscommon.WhiteListHandlerStub{}, - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - ArgumentsParser: &testscommon.ArgumentParserMock{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 400, - MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, + CoreComponents: coreComp, + CryptoComponents: cryptoComp, + Accounts: &stateMock.AccountsStub{}, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + MainMessenger: &mock.TopicHandlerStub{}, + FullArchiveMessenger: &mock.TopicHandlerStub{}, + Store: createShardStore(), + DataPool: createShardDataPools(), + MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: &testscommon.TimeCacheStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + SizeCheckDelta: 0, + ValidityAttester: &mock.ValidityAttesterStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + ArgumentsParser: &testscommon.ArgumentParserMock{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/process/interceptors/processor/chunk/chunk.go b/process/interceptors/processor/chunk/chunk.go index 9edf585d88c..96e7b8742a4 100644 --- a/process/interceptors/processor/chunk/chunk.go +++ b/process/interceptors/processor/chunk/chunk.go @@ -1,25 +1,29 @@ package chunk import ( + "time" + logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("process/interceptors/processor") type chunk struct { - reference []byte - maxChunks uint32 - data map[uint32][]byte - size int + reference []byte + maxChunks uint32 + data map[uint32][]byte + size int + lastUpdated time.Time } // NewChunk creates a new chunk instance able to account for the existing and missing chunks of a larger buffer // Not a concurrent safe component func NewChunk(maxChunks uint32, reference []byte) *chunk { return &chunk{ - reference: reference, - data: make(map[uint32][]byte), - maxChunks: maxChunks, + reference: reference, + data: make(map[uint32][]byte), + maxChunks: maxChunks, + lastUpdated: time.Now(), } } @@ -32,6 +36,7 @@ func (c *chunk) Put(chunkIndex uint32, buff []byte) { existing := c.data[chunkIndex] c.data[chunkIndex] = buff c.size = c.size - len(existing) + len(buff) + c.lastUpdated = time.Now() } // TryAssembleAllChunks will try to assemble the original payload by iterating all available chunks @@ -72,6 +77,11 @@ func (c *chunk) MaxChunks() uint32 { return c.maxChunks } +// LastUpdated returns the time when the latest chunk was recorded for this payload. +func (c *chunk) LastUpdated() time.Time { + return c.lastUpdated +} + // Size returns the size in bytes stored in the values of the inner map func (c *chunk) Size() int { return c.size diff --git a/process/interceptors/processor/trieNodeChunksProcessor.go b/process/interceptors/processor/trieNodeChunksProcessor.go index 475017796bc..6d4e1c71cc0 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor.go +++ b/process/interceptors/processor/trieNodeChunksProcessor.go @@ -20,6 +20,7 @@ type chunkHandler interface { TryAssembleAllChunks() []byte GetAllMissingChunkIndexes() []uint32 MaxChunks() uint32 + LastUpdated() time.Time Size() int IsInterfaceNil() bool } @@ -31,24 +32,26 @@ type checkRequest struct { // TrieNodesChunksProcessorArgs is the argument DTO used in the trieNodeChunksProcessor constructor type TrieNodesChunksProcessorArgs struct { - Hasher hashing.Hasher - ChunksCacher storage.Cacher - RequestInterval time.Duration - RequestHandler process.RequestHandler - Topic string - MaxAllowedChunks uint32 + Hasher hashing.Hasher + ChunksCacher storage.Cacher + RequestInterval time.Duration + RequestHandler process.RequestHandler + Topic string + MaxAllowedChunks uint32 + ChunkInactivityTimeout time.Duration } type trieNodeChunksProcessor struct { - hasher hashing.Hasher - chunksCacher storage.Cacher - chanCheckRequests chan checkRequest - requestInterval time.Duration - requestHandler process.RequestHandler - topic string - maxAllowedChunks uint32 - cancel func() - chanClose chan struct{} + hasher hashing.Hasher + chunksCacher storage.Cacher + chanCheckRequests chan checkRequest + requestInterval time.Duration + requestHandler process.RequestHandler + topic string + maxAllowedChunks uint32 + chunkInactivityTimeout time.Duration + cancel func() + chanClose chan struct{} } // NewTrieNodeChunksProcessor creates a new trieNodeChunksProcessor instance @@ -72,16 +75,20 @@ func NewTrieNodeChunksProcessor(arg TrieNodesChunksProcessorArgs) (*trieNodeChun if arg.MaxAllowedChunks < 2 { return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor, MaxAllowedChunks should be at least 2", process.ErrInvalidValue) } + if arg.ChunkInactivityTimeout <= 0 { + return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor, ChunkInactivityTimeout should be greater than 0", process.ErrInvalidValue) + } tncp := &trieNodeChunksProcessor{ - hasher: arg.Hasher, - chunksCacher: arg.ChunksCacher, - chanCheckRequests: make(chan checkRequest), - requestInterval: arg.RequestInterval, - requestHandler: arg.RequestHandler, - topic: arg.Topic, - maxAllowedChunks: arg.MaxAllowedChunks, - chanClose: make(chan struct{}), + hasher: arg.Hasher, + chunksCacher: arg.ChunksCacher, + chanCheckRequests: make(chan checkRequest), + requestInterval: arg.RequestInterval, + requestHandler: arg.RequestHandler, + topic: arg.Topic, + maxAllowedChunks: arg.MaxAllowedChunks, + chunkInactivityTimeout: arg.ChunkInactivityTimeout, + chanClose: make(chan struct{}), } var ctx context.Context ctx, tncp.cancel = context.WithCancel(context.Background()) @@ -249,6 +256,15 @@ func (proc *trieNodeChunksProcessor) requestMissingForReference(reference []byte proc.chunksCacher.Remove(reference) return } + if time.Since(chunkData.LastUpdated()) > proc.chunkInactivityTimeout { + log.Warn("dropping stale trie node chunk tracker after inactivity timeout", + "reference", reference, + "lastUpdated", chunkData.LastUpdated(), + "inactivityTimeout", proc.chunkInactivityTimeout, + ) + proc.chunksCacher.Remove(reference) + return + } missing := chunkData.GetAllMissingChunkIndexes() for _, missingChunkIndex := range missing { diff --git a/process/interceptors/processor/trieNodeChunksProcessor_test.go b/process/interceptors/processor/trieNodeChunksProcessor_test.go index bdda540fd8b..c62c418c01f 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor_test.go +++ b/process/interceptors/processor/trieNodeChunksProcessor_test.go @@ -36,11 +36,12 @@ func createMockTrieNodesChunksProcessorArgs() TrieNodesChunksProcessorArgs { return 32 }, }, - ChunksCacher: cache.NewCacherMock(), - RequestInterval: time.Second, - RequestHandler: &testscommon.RequestHandlerStub{}, - Topic: "topic", - MaxAllowedChunks: 3, + ChunksCacher: cache.NewCacherMock(), + RequestInterval: time.Second, + RequestHandler: &testscommon.RequestHandlerStub{}, + Topic: "topic", + MaxAllowedChunks: 3, + ChunkInactivityTimeout: 10 * time.Second, } } @@ -104,6 +105,16 @@ func TestNewTrieNodeChunksProcessor_InvalidMaxAllowedChunks(t *testing.T) { assert.True(t, check.IfNil(tncp)) } +func TestNewTrieNodeChunksProcessor_InvalidChunkInactivityTimeout(t *testing.T) { + t.Parallel() + + args := createMockTrieNodesChunksProcessorArgs() + args.ChunkInactivityTimeout = 0 + tncp, err := NewTrieNodeChunksProcessor(args) + assert.True(t, errors.Is(err, process.ErrInvalidValue)) + assert.True(t, check.IfNil(tncp)) +} + func TestNewTrieNodeChunksProcessor_ShouldWork(t *testing.T) { t.Parallel() @@ -323,6 +334,32 @@ func TestTrieNodeChunksProcessor_RequestMissingForReferenceShouldDropCachedChunk _ = tncp.Close() } +func TestTrieNodeChunksProcessor_RequestMissingForReferenceShouldDropStaleCachedChunk(t *testing.T) { + t.Parallel() + + args := createMockTrieNodesChunksProcessorArgs() + args.ChunkInactivityTimeout = 10 * time.Millisecond + numRequested := uint32(0) + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestTrieNodeCalled: func(_ []byte, _ string, _ uint32) { + atomic.AddUint32(&numRequested, 1) + }, + } + + tncp, _ := NewTrieNodeChunksProcessor(args) + staleChunk := chunk.NewChunk(args.MaxAllowedChunks, reference) + staleChunk.Put(0, []byte("buff1")) + args.ChunksCacher.Put(reference, staleChunk, staleChunk.Size()) + + time.Sleep(args.ChunkInactivityTimeout + 5*time.Millisecond) + tncp.requestMissingForReference(reference, context.Background()) + + assert.Equal(t, 0, args.ChunksCacher.Len()) + assert.Equal(t, uint32(0), atomic.LoadUint32(&numRequested)) + + _ = tncp.Close() +} + func TestTrieNodeChunksProcessor_CheckBatchComponentClosed(t *testing.T) { t.Parallel() diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index 2b546238f0c..5153b670970 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -386,9 +386,10 @@ func GetGeneralConfig() config.Config { CheckNodesOnDisk: false, }, Antiflood: config.AntifloodConfig{ - NumConcurrentResolverJobs: 2, - NumConcurrentResolvingTrieNodesJobs: 1, - MaxAllowedTrieNodeChunks: 400, + NumConcurrentResolverJobs: 2, + NumConcurrentResolvingTrieNodesJobs: 1, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeoutInSec: 10, TxAccumulator: config.TxAccumulatorConfig{ MaxAllowedTimeInMilliseconds: 10, MaxDeviationTimeInMilliseconds: 1, From 6251fad4df549878bc2c76d38f88235e30b0e6e5 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 29 Apr 2026 18:16:29 +0300 Subject: [PATCH 007/116] add resolver exception recover --- .../resolvers/equivalentProofsResolver.go | 13 ++++- dataRetriever/resolvers/headerResolver.go | 15 ++++-- dataRetriever/resolvers/miniblockResolver.go | 12 ++++- .../resolvers/peerAuthenticationResolver.go | 12 ++++- .../resolvers/transactionResolver.go | 12 ++++- dataRetriever/resolvers/trieNodeResolver.go | 13 ++++- .../resolvers/trieNodeResolver_test.go | 51 +++++++++++++++++++ .../resolvers/validatorInfoResolver.go | 12 ++++- 8 files changed, 125 insertions(+), 15 deletions(-) diff --git a/dataRetriever/resolvers/equivalentProofsResolver.go b/dataRetriever/resolvers/equivalentProofsResolver.go index c36c3e9ac92..b2e51da87b4 100644 --- a/dataRetriever/resolvers/equivalentProofsResolver.go +++ b/dataRetriever/resolvers/equivalentProofsResolver.go @@ -2,6 +2,8 @@ package resolvers import ( "fmt" + "runtime/debug" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" @@ -90,8 +92,15 @@ func checkArgEquivalentProofsResolver(args ArgEquivalentProofsResolver) error { // ProcessReceivedMessage represents the callback func from the p2p.Messenger that is called each time a new message is received // (for the topic this validator was registered to, usually a request topic) -func (res *equivalentProofsResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := res.canProcessMessage(message, fromConnectedPeer) +func (res *equivalentProofsResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in equivalentProofsResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = res.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/headerResolver.go b/dataRetriever/resolvers/headerResolver.go index dbd8626bf3a..99eb50dbde0 100644 --- a/dataRetriever/resolvers/headerResolver.go +++ b/dataRetriever/resolvers/headerResolver.go @@ -1,12 +1,14 @@ package resolvers import ( + "fmt" + "runtime/debug" "sync" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/typeConverters" - "github.com/multiversx/mx-chain-logger-go" + logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/resolvers/epochproviders/disabled" @@ -110,8 +112,15 @@ func (hdrRes *HeaderResolver) SetEpochHandler(epochHandler dataRetriever.EpochHa // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (hdrRes *HeaderResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := hdrRes.canProcessMessage(message, fromConnectedPeer) +func (hdrRes *HeaderResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in HeaderResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = hdrRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/miniblockResolver.go b/dataRetriever/resolvers/miniblockResolver.go index 3fb74105af5..0909a2efb47 100644 --- a/dataRetriever/resolvers/miniblockResolver.go +++ b/dataRetriever/resolvers/miniblockResolver.go @@ -2,6 +2,7 @@ package resolvers import ( "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -78,8 +79,15 @@ func checkArgMiniblockResolver(arg ArgMiniblockResolver) error { // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (mbRes *miniblockResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := mbRes.canProcessMessage(message, fromConnectedPeer) +func (mbRes *miniblockResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in miniblockResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = mbRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/peerAuthenticationResolver.go b/dataRetriever/resolvers/peerAuthenticationResolver.go index 49f29ff0246..14091ca7698 100644 --- a/dataRetriever/resolvers/peerAuthenticationResolver.go +++ b/dataRetriever/resolvers/peerAuthenticationResolver.go @@ -2,6 +2,7 @@ package resolvers import ( "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -76,8 +77,15 @@ func checkArgPeerAuthenticationResolver(arg ArgPeerAuthenticationResolver) error // ProcessReceivedMessage represents the callback func from the p2p.Messenger that is called each time a new message is received // (for the topic this validator was registered to, usually a request topic) -func (res *peerAuthenticationResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := res.canProcessMessage(message, fromConnectedPeer) +func (res *peerAuthenticationResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in peerAuthenticationResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = res.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/transactionResolver.go b/dataRetriever/resolvers/transactionResolver.go index 8495c970a70..4d79277f260 100644 --- a/dataRetriever/resolvers/transactionResolver.go +++ b/dataRetriever/resolvers/transactionResolver.go @@ -2,6 +2,7 @@ package resolvers import ( "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -83,8 +84,15 @@ func checkArgTxResolver(arg ArgTxResolver) error { // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (txRes *TxResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := txRes.canProcessMessage(message, fromConnectedPeer) +func (txRes *TxResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in TxResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = txRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/trieNodeResolver.go b/dataRetriever/resolvers/trieNodeResolver.go index 78ed24d0159..721105b64d7 100644 --- a/dataRetriever/resolvers/trieNodeResolver.go +++ b/dataRetriever/resolvers/trieNodeResolver.go @@ -1,6 +1,8 @@ package resolvers import ( + "fmt" + "runtime/debug" "sync" "github.com/multiversx/mx-chain-core-go/core" @@ -63,8 +65,15 @@ func checkArgTrieNodeResolver(arg ArgTrieNodeResolver) error { // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (tnRes *TrieNodeResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := tnRes.canProcessMessage(message, fromConnectedPeer) +func (tnRes *TrieNodeResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in TrieNodeResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = tnRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/trieNodeResolver_test.go b/dataRetriever/resolvers/trieNodeResolver_test.go index b988b2f2959..6347f02875a 100644 --- a/dataRetriever/resolvers/trieNodeResolver_test.go +++ b/dataRetriever/resolvers/trieNodeResolver_test.go @@ -587,6 +587,57 @@ func TestTrieNodeResolver_ProcessReceivedMessageLargeTrieNodeShouldSendFirstChun testTrieNodeResolverProcessReceivedMessageLargeTrieNode(t, randBuff, 0, 4, 0, core.MaxBufferSizeToSendTrieNodes) } +func TestTrieNodeResolver_ProcessReceivedMessageLargeTrieNodeMaxChunkIndex(t *testing.T) { + t.Parallel() + + largeBuffer := make([]byte, 393216) // 256k + 128k + chunkIndex := uint32(2) + + nodes := [][]byte{largeBuffer} + hashes := [][]byte{[]byte("hash1")} + + sendWasCalled := false + arg := createMockArgTrieNodeResolver() + arg.SenderResolver = &mock.TopicResolverSenderStub{ + SendCalled: func(buff []byte, peer core.PeerID, source p2p.MessageHandler) error { + sendWasCalled = true + return nil + }, + } + arg.TrieDataGetter = &trieMock.TrieStub{ + GetSerializedNodeCalled: func(hash []byte) ([]byte, error) { + for i := 0; i < len(hashes); i++ { + if bytes.Equal(hash, hashes[i]) { + return nodes[i], nil + } + } + + return nil, fmt.Errorf("not found") + }, + GetSerializedNodesCalled: func(i []byte, u uint64) ([][]byte, uint64, error) { + return make([][]byte, 0), 0, nil + }, + } + tnRes, _ := resolvers.NewTrieNodeResolver(arg) + + data, _ := arg.Marshaller.Marshal( + &dataRetriever.RequestData{ + Type: dataRetriever.HashType, + Value: []byte("hash1"), + ChunkIndex: chunkIndex, + }, + ) + msg := &p2pmocks.P2PMessageMock{DataField: data} + + msgID, err := tnRes.ProcessReceivedMessage(msg, fromConnectedPeer, &p2pmocks.MessengerStub{}) + assert.Nil(t, err) + require.False(t, sendWasCalled) + assert.Len(t, msgID, 0) + + assert.True(t, arg.Throttler.(*mock.ThrottlerStub).StartWasCalled()) + assert.True(t, arg.Throttler.(*mock.ThrottlerStub).EndWasCalled()) +} + func TestTrieNodeResolver_ProcessReceivedMessageLargeTrieNodeShouldSendRequiredChunk(t *testing.T) { t.Parallel() diff --git a/dataRetriever/resolvers/validatorInfoResolver.go b/dataRetriever/resolvers/validatorInfoResolver.go index 65255b8ad8f..f3cb291abef 100644 --- a/dataRetriever/resolvers/validatorInfoResolver.go +++ b/dataRetriever/resolvers/validatorInfoResolver.go @@ -3,6 +3,7 @@ package resolvers import ( "encoding/hex" "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -90,8 +91,15 @@ func checkArgs(args ArgValidatorInfoResolver) error { // ProcessReceivedMessage represents the callback func from the p2p.Messenger that is called each time a new message is received // (for the topic this validator was registered to, usually a request topic) -func (res *validatorInfoResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := res.canProcessMessage(message, fromConnectedPeer) +func (res *validatorInfoResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in validatorInfoResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = res.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } From 41f2050fb06c7d252bc8091ed71b6d5566cbbd86 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 29 Apr 2026 18:28:54 +0300 Subject: [PATCH 008/116] fix ai finding --- consensus/spos/consensusMessageValidator.go | 30 +++++++++++++++---- .../spos/consensusMessageValidator_test.go | 2 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/consensus/spos/consensusMessageValidator.go b/consensus/spos/consensusMessageValidator.go index c2a63264c75..5bc6bc52520 100644 --- a/consensus/spos/consensusMessageValidator.go +++ b/consensus/spos/consensusMessageValidator.go @@ -201,6 +201,7 @@ func (cmv *consensusMessageValidator) checkConsensusMessageValidity(cnsMsg *cons err = cmv.peerSignatureHandler.VerifyPeerSignature(cnsMsg.PubKey, core.PeerID(cnsMsg.OriginatorPid), cnsMsg.Signature) if err != nil { + cmv.removeMessageTypeToPublicKey(cnsMsg.PubKey, cnsMsg.RoundIndex, msgType) return fmt.Errorf("%w : verify signature for received message from consensus topic failed: %s", ErrInvalidSignature, err.Error()) @@ -208,12 +209,11 @@ func (cmv *consensusMessageValidator) checkConsensusMessageValidity(cnsMsg *cons cnsMsgOriginator := core.PeerID(cnsMsg.OriginatorPid) if cnsMsgOriginator != originator { + cmv.removeMessageTypeToPublicKey(cnsMsg.PubKey, cnsMsg.RoundIndex, msgType) return fmt.Errorf("%w : pubsub originator pid: %s, cnsMsg.OriginatorPid: %s", ErrOriginatorMismatch, p2p.PeerIdToShortString(originator), p2p.PeerIdToShortString(cnsMsgOriginator)) } - cmv.addMessageTypeToPublicKey(cnsMsg.PubKey, cnsMsg.RoundIndex, msgType) - return nil } @@ -500,21 +500,25 @@ func (cmv *consensusMessageValidator) isMessageTypeLimitReached(pk []byte, round mapMsgType, ok := cmv.mapPkConsensusMessages[key] if !ok { + cmv.addMessageTypeToPublicKey(pk, round, msgType) return false } numMsgType, ok := mapMsgType[msgType] if !ok { + cmv.addMessageTypeToPublicKey(pk, round, msgType) return false } - return numMsgType >= cmv.consensusService.GetMaxNumOfMessageTypeAccepted(msgType) + isLimitReached := numMsgType >= cmv.consensusService.GetMaxNumOfMessageTypeAccepted(msgType) + if !isLimitReached { + cmv.addMessageTypeToPublicKey(pk, round, msgType) + } + + return isLimitReached } func (cmv *consensusMessageValidator) addMessageTypeToPublicKey(pk []byte, round int64, msgType consensus.MessageType) { - cmv.mutPkConsensusMessages.Lock() - defer cmv.mutPkConsensusMessages.Unlock() - key := fmt.Sprintf("%s_%d", string(pk), round) mapMsgType, ok := cmv.mapPkConsensusMessages[key] @@ -526,6 +530,20 @@ func (cmv *consensusMessageValidator) addMessageTypeToPublicKey(pk []byte, round mapMsgType[msgType]++ } +func (cmv *consensusMessageValidator) removeMessageTypeToPublicKey(pk []byte, round int64, msgType consensus.MessageType) { + cmv.mutPkConsensusMessages.RLock() + defer cmv.mutPkConsensusMessages.RUnlock() + + key := fmt.Sprintf("%s_%d", string(pk), round) + + mapMsgType, ok := cmv.mapPkConsensusMessages[key] + if !ok { + return + } + + mapMsgType[msgType]-- +} + func (cmv *consensusMessageValidator) resetConsensusMessages() { cmv.mutPkConsensusMessages.Lock() cmv.mapPkConsensusMessages = make(map[string]map[consensus.MessageType]uint32) diff --git a/consensus/spos/consensusMessageValidator_test.go b/consensus/spos/consensusMessageValidator_test.go index 9936694d21f..d247d2cfeff 100644 --- a/consensus/spos/consensusMessageValidator_test.go +++ b/consensus/spos/consensusMessageValidator_test.go @@ -905,7 +905,7 @@ func TestIsMessageTypeLimitReached_ShouldWork(t *testing.T) { cmv.AddMessageTypeToPublicKey([]byte("pk1"), 1, bls.MtBlockHeader) - assert.False(t, cmv.IsMessageTypeLimitReached([]byte("pk1"), 1, bls.MtBlockBody)) + assert.True(t, cmv.IsMessageTypeLimitReached([]byte("pk1"), 1, bls.MtBlockBody)) assert.True(t, cmv.IsMessageTypeLimitReached([]byte("pk1"), 1, bls.MtBlockHeader)) assert.False(t, cmv.IsMessageTypeLimitReached([]byte("pk1"), 2, bls.MtBlockHeader)) } From 819d4552a114a24f88835506a09b70bfd76635e4 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 30 Apr 2026 12:01:00 +0300 Subject: [PATCH 009/116] replace hard-coded val with const --- .../interceptors/processor/trieNodeChunksProcessor.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/process/interceptors/processor/trieNodeChunksProcessor.go b/process/interceptors/processor/trieNodeChunksProcessor.go index 6d4e1c71cc0..4f1176a7b27 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor.go +++ b/process/interceptors/processor/trieNodeChunksProcessor.go @@ -13,7 +13,10 @@ import ( "github.com/multiversx/mx-chain-go/storage" ) -const minimumRequestTimeInterval = time.Millisecond * 200 +const ( + minimumRequestTimeInterval = time.Millisecond * 200 + minNumChunks = 2 +) type chunkHandler interface { Put(chunkIndex uint32, buff []byte) @@ -72,8 +75,8 @@ func NewTrieNodeChunksProcessor(arg TrieNodesChunksProcessorArgs) (*trieNodeChun if len(arg.Topic) == 0 { return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor", process.ErrEmptyTopic) } - if arg.MaxAllowedChunks < 2 { - return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor, MaxAllowedChunks should be at least 2", process.ErrInvalidValue) + if arg.MaxAllowedChunks < minNumChunks { + return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor, MaxAllowedChunks should be at least %v", process.ErrInvalidValue, minNumChunks) } if arg.ChunkInactivityTimeout <= 0 { return nil, fmt.Errorf("%w in NewTrieNodeChunksProcessor, ChunkInactivityTimeout should be greater than 0", process.ErrInvalidValue) From 7cb8f54a7b264a98dcb63db20b2cf1fd87e0e929 Mon Sep 17 00:00:00 2001 From: miiu Date: Thu, 30 Apr 2026 13:42:41 +0300 Subject: [PATCH 010/116] bloom filter --- cmd/node/config/config.toml | 33 +++++++++++++++++++ config/config.go | 17 ++++++---- go.mod | 2 +- go.sum | 4 +-- .../benchmarks/loadFromTrie_test.go | 2 +- .../storage/storagePutRemove_test.go | 2 +- integrationTests/testStorage.go | 4 +-- storage/database/db.go | 8 ++--- storage/database/db_test.go | 8 ++--- storage/factory/persisterCreator.go | 11 ++++--- storage/pruning/pruningStorer_test.go | 2 +- 11 files changed, 65 insertions(+), 28 deletions(-) diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index ed93fbb82a2..ac55f09a126 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -119,6 +119,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [ReceiptsStorage] [ReceiptsStorage.Cache] @@ -132,6 +133,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [ScheduledSCRsStorage] [ScheduledSCRsStorage.Cache] @@ -145,6 +147,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [PeerBlockBodyStorage] [PeerBlockBodyStorage.Cache] @@ -158,6 +161,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [BlockHeaderStorage] [BlockHeaderStorage.Cache] @@ -171,6 +175,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [BootstrapStorage] [BootstrapStorage.Cache] @@ -184,6 +189,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [MetaBlockStorage] [MetaBlockStorage.Cache] @@ -197,6 +203,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [ProofsStorage] [ProofsStorage.Cache] @@ -210,6 +217,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [TxStorage] [TxStorage.Cache] @@ -223,6 +231,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 30000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [UnsignedTransactionStorage] [UnsignedTransactionStorage.Cache] @@ -236,6 +245,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [RewardTxStorage] [RewardTxStorage.Cache] @@ -249,6 +259,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [SmartContractsStorage] [SmartContractsStorage.Cache] @@ -262,6 +273,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [SmartContractsStorageSimulate] [SmartContractsStorageSimulate.Cache] @@ -275,6 +287,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [SmartContractsStorageForSCQuery] [SmartContractsStorageForSCQuery.Cache] @@ -288,6 +301,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [StatusMetricsStorage] [StatusMetricsStorage.Cache] @@ -300,6 +314,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [TrieEpochRootHashStorage] [TrieEpochRootHashStorage.Cache] @@ -313,6 +328,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 500 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [ShardHdrNonceHashStorage] [ShardHdrNonceHashStorage.Cache] @@ -326,6 +342,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [MetaHdrNonceHashStorage] [MetaHdrNonceHashStorage.Cache] @@ -339,6 +356,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [AccountsTrieStorage] [AccountsTrieStorage.Cache] @@ -367,6 +385,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [PeerAccountsTrieStorage] [PeerAccountsTrieStorage.Cache] @@ -393,6 +412,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [TrieStorageManagerConfig] PruningBufferLen = 100000 @@ -530,6 +550,7 @@ MaxBatchSize = 45000 MaxOpenFiles = 10 UseTmpAsFilePath = true + BloomFilterBtsPerKey = 10 [Antiflood] Enabled = true @@ -780,6 +801,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [Hardfork.ExportKeysStorageConfig] [Hardfork.ExportKeysStorageConfig.Cache] Name = "HardFork.ExportKeysStorageConfig" @@ -791,6 +813,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [Hardfork.ExportTriesStorageConfig] [Hardfork.ExportTriesStorageConfig.Cache] Name = "HardFork.ExportTriesStorageConfig" @@ -802,6 +825,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [Hardfork.ImportStateStorageConfig] [Hardfork.ImportStateStorageConfig.Cache] Name = "HardFork.ImportStateStorageConfig" @@ -813,6 +837,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [Hardfork.ImportKeysStorageConfig] [Hardfork.ImportKeysStorageConfig.Cache] Name = "HardFork.ImportKeysStorageConfig" @@ -824,6 +849,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [Debug] [Debug.InterceptorResolver] @@ -882,6 +908,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [DbLookupExtensions] Enabled = false @@ -896,6 +923,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [DbLookupExtensions.MiniblockHashByTxHashStorageConfig.Cache] Name = "DbLookupExtensions.MiniblockHashByTxHashStorage" Capacity = 20000 @@ -906,6 +934,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [DbLookupExtensions.EpochByHashStorageConfig.Cache] Name = "DbLookupExtensions.EpochByHashStorage" Capacity = 20000 @@ -916,6 +945,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [DbLookupExtensions.ResultsHashesByTxHashStorageConfig.Cache] Name = "DbLookupExtensions.ResultsHashesByTxHashStorage" Capacity = 20000 @@ -926,6 +956,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [DbLookupExtensions.ESDTSuppliesStorageConfig.Cache] Name = "DbLookupExtensions.ESDTSuppliesStorage" Capacity = 20000 @@ -936,6 +967,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [DbLookupExtensions.RoundHashStorageConfig.Cache] Name = "DbLookupExtensions.RoundHashStorage" Capacity = 20000 @@ -946,6 +978,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 + BloomFilterBtsPerKey = 10 [Logs] LogFileLifeSpanInMB = 1024 # 1GB diff --git a/config/config.go b/config/config.go index 0ad08a34111..90ed76db1ed 100644 --- a/config/config.go +++ b/config/config.go @@ -35,6 +35,9 @@ type DBConfig struct { UseTmpAsFilePath bool ShardIDProviderType string NumShards int32 + // BloomFilterBtsPerKey == 0, the Bloom filter is disabled. + // Otherwise, it specifies the number of bits per key used by the Bloom filter. + BloomFilterBtsPerKey int } // StorageConfig will map the storage unit configuration @@ -169,14 +172,14 @@ type Config struct { MetaBlockStorage StorageConfig ProofsStorage StorageConfig - AccountsTrieStorage StorageConfig - PeerAccountsTrieStorage StorageConfig - EvictionWaitingList EvictionWaitingListConfig - StateTriesConfig StateTriesConfig + AccountsTrieStorage StorageConfig + PeerAccountsTrieStorage StorageConfig + EvictionWaitingList EvictionWaitingListConfig + StateTriesConfig StateTriesConfig StateAccessesCollectorConfig StateAccessesCollectorConfig - TrieStorageManagerConfig TrieStorageManagerConfig - TrieLeavesRetrieverConfig TrieLeavesRetrieverConfig - BadBlocksCache CacheConfig + TrieStorageManagerConfig TrieStorageManagerConfig + TrieLeavesRetrieverConfig TrieLeavesRetrieverConfig + BadBlocksCache CacheConfig TxBlockBodyDataPool CacheConfig PeerBlockBodyDataPool CacheConfig diff --git a/go.mod b/go.mod index 82f01ad56b5..f9a0ebcd792 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.9.2 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 - github.com/multiversx/mx-chain-storage-go v1.1.0 + github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260429132446-df76d7a4bc38 github.com/multiversx/mx-chain-vm-common-go v1.6.0 github.com/multiversx/mx-chain-vm-go v1.5.43 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 diff --git a/go.sum b/go.sum index f340fdf22d3..44f4e7782b6 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= -github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260429132446-df76d7a4bc38 h1:tInQvnDEq/fY6VomIelQd8IHBbBX9X3wBNPk1s3VMrw= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260429132446-df76d7a4bc38/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go v1.6.0 h1:M2zmf/ptEINciWxYCPLIkwOMTvvzWjELYYB+0MMQ5Gw= github.com/multiversx/mx-chain-vm-common-go v1.6.0/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.43 h1:bMC6aAv0T8BFDPQlc0kSnZtv413NOlF+le5zluMmNQY= diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index 8b2d2736b1a..c9c051a7f84 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -156,7 +156,7 @@ func getNewTrieStorage() storage.Storer { maxBatchSize := 40000 maxNumOpenedFiles := 10 - db, _ := database.NewSerialDB("AccountsTrie", batchDelaySeconds, maxBatchSize, maxNumOpenedFiles) + db, _ := database.NewSerialDB("AccountsTrie", batchDelaySeconds, maxBatchSize, maxNumOpenedFiles, 0) cacher, _ := storageunit.NewCache(storageunit.CacheConfig{ Type: storageunit.SizeLRUCache, Capacity: 1, diff --git a/integrationTests/longTests/storage/storagePutRemove_test.go b/integrationTests/longTests/storage/storagePutRemove_test.go index a10d0085ffc..f89459b96db 100644 --- a/integrationTests/longTests/storage/storagePutRemove_test.go +++ b/integrationTests/longTests/storage/storagePutRemove_test.go @@ -20,7 +20,7 @@ func TestPutRemove(t *testing.T) { cache, _ := storageunit.NewCache(storageunit.CacheConfig{Type: storageunit.LRUCache, Capacity: 5000, Shards: 16, SizeInBytes: 0}) dir := t.TempDir() log.Info("opened in", "directory", dir) - lvdb1, err := database.NewLevelDB(dir, 2, 1000, 10) + lvdb1, err := database.NewLevelDB(dir, 2, 1000, 10, 10) assert.NoError(t, err) defer func() { diff --git a/integrationTests/testStorage.go b/integrationTests/testStorage.go index 567b3b9a349..b477927c566 100644 --- a/integrationTests/testStorage.go +++ b/integrationTests/testStorage.go @@ -77,7 +77,7 @@ func (ts *TestStorage) CreateStoredData(nonce uint64) ([]byte, []byte) { // CreateStorageLevelDB creates a storage levelDB func (ts *TestStorage) CreateStorageLevelDB() storage.Storer { - db, _ := database.NewLevelDB("Transactions", batchDelaySeconds, maxBatchSize, maxOpenFiles) + db, _ := database.NewLevelDB("Transactions", batchDelaySeconds, maxBatchSize, maxOpenFiles, 0) cacher, _ := cache.NewLRUCache(50000) store, _ := storageunit.NewStorageUnit( cacher, @@ -89,7 +89,7 @@ func (ts *TestStorage) CreateStorageLevelDB() storage.Storer { // CreateStorageLevelDBSerial creates a storage levelDB serial func (ts *TestStorage) CreateStorageLevelDBSerial() storage.Storer { - db, _ := database.NewSerialDB("Transactions", batchDelaySeconds, maxBatchSize, maxOpenFiles) + db, _ := database.NewSerialDB("Transactions", batchDelaySeconds, maxBatchSize, maxOpenFiles, 0) cacher, _ := cache.NewLRUCache(50000) store, _ := storageunit.NewStorageUnit( cacher, diff --git a/storage/database/db.go b/storage/database/db.go index 7e677ed954c..52c720d572f 100644 --- a/storage/database/db.go +++ b/storage/database/db.go @@ -23,14 +23,14 @@ func NewlruDB(size uint32) (storage.Persister, error) { // NewLevelDB is a constructor for the leveldb persister // It creates the files in the location given as parameter -func NewLevelDB(path string, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int) (s *leveldb.DB, err error) { - return leveldb.NewDB(path, batchDelaySeconds, maxBatchSize, maxOpenFiles) +func NewLevelDB(path string, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int, bloomFilterSize int) (s *leveldb.DB, err error) { + return leveldb.NewDB(path, batchDelaySeconds, maxBatchSize, maxOpenFiles, bloomFilterSize) } // NewSerialDB is a constructor for the leveldb persister // It creates the files in the location given as parameter -func NewSerialDB(path string, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int) (s *leveldb.SerialDB, err error) { - return leveldb.NewSerialDB(path, batchDelaySeconds, maxBatchSize, maxOpenFiles) +func NewSerialDB(path string, batchDelaySeconds int, maxBatchSize int, maxOpenFiles int, bloomFilterSize int) (s *leveldb.SerialDB, err error) { + return leveldb.NewSerialDB(path, batchDelaySeconds, maxBatchSize, maxOpenFiles, bloomFilterSize) } // NewShardIDProvider is a constructor for shard id provider diff --git a/storage/database/db_test.go b/storage/database/db_test.go index d04aaa2a78d..1909ff53c6b 100644 --- a/storage/database/db_test.go +++ b/storage/database/db_test.go @@ -38,14 +38,14 @@ func TestNewLevelDB(t *testing.T) { t.Run("invalid argument should error", func(t *testing.T) { t.Parallel() - instance, err := NewLevelDB(t.TempDir(), 0, 0, 0) + instance, err := NewLevelDB(t.TempDir(), 0, 0, 0, 0) assert.Nil(t, instance) assert.NotNil(t, err) }) t.Run("should work", func(t *testing.T) { t.Parallel() - instance, err := NewLevelDB(t.TempDir(), 1, 1, 1) + instance, err := NewLevelDB(t.TempDir(), 1, 1, 1, 0) assert.NotNil(t, instance) assert.Nil(t, err) _ = instance.Close() @@ -58,14 +58,14 @@ func TestNewSerialDB(t *testing.T) { t.Run("invalid argument should error", func(t *testing.T) { t.Parallel() - instance, err := NewSerialDB(t.TempDir(), 0, 0, 0) + instance, err := NewSerialDB(t.TempDir(), 0, 0, 0, 0) assert.Nil(t, instance) assert.NotNil(t, err) }) t.Run("should work", func(t *testing.T) { t.Parallel() - instance, err := NewSerialDB(t.TempDir(), 1, 1, 1) + instance, err := NewSerialDB(t.TempDir(), 1, 1, 1, 0) assert.NotNil(t, instance) assert.Nil(t, err) _ = instance.Close() diff --git a/storage/factory/persisterCreator.go b/storage/factory/persisterCreator.go index 0d17287815e..c9f49eeb72e 100644 --- a/storage/factory/persisterCreator.go +++ b/storage/factory/persisterCreator.go @@ -43,11 +43,12 @@ func (pc *persisterCreator) CreateBasePersister(path string) (storage.Persister, var dbType = storageunit.DBType(pc.conf.Type) argsDB := factory.ArgDB{ - DBType: dbType, - Path: path, - BatchDelaySeconds: pc.conf.BatchDelaySeconds, - MaxBatchSize: pc.conf.MaxBatchSize, - MaxOpenFiles: pc.conf.MaxOpenFiles, + DBType: dbType, + Path: path, + BatchDelaySeconds: pc.conf.BatchDelaySeconds, + MaxBatchSize: pc.conf.MaxBatchSize, + MaxOpenFiles: pc.conf.MaxOpenFiles, + BloomFilterBtsPerKey: pc.conf.BloomFilterBtsPerKey, } return storageunit.NewDB(argsDB) diff --git a/storage/pruning/pruningStorer_test.go b/storage/pruning/pruningStorer_test.go index 248cc53cda2..2e7801ce6e4 100644 --- a/storage/pruning/pruningStorer_test.go +++ b/storage/pruning/pruningStorer_test.go @@ -98,7 +98,7 @@ func getDefaultArgsSerialDB() pruning.StorerArgs { cacheConf.Capacity = 40 persisterFactory := &mock.PersisterFactoryStub{ CreateCalled: func(path string) (storage.Persister, error) { - return database.NewSerialDB(path, 1, 20, 10) + return database.NewSerialDB(path, 1, 20, 10, 10) }, } pathManager := &testscommon.PathManagerStub{PathForEpochCalled: func(shardId string, epoch uint32, identifier string) string { From a1637fe38329c47e2bffe5bf9124198e01b18e8e Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Thu, 30 Apr 2026 14:41:23 +0300 Subject: [PATCH 011/116] verify padding bits --- common/common.go | 11 ++++++++++- common/errors.go | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/common/common.go b/common/common.go index 6641777b54a..fb8650de073 100644 --- a/common/common.go +++ b/common/common.go @@ -9,8 +9,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-go/config" logger "github.com/multiversx/mx-chain-logger-go" + + "github.com/multiversx/mx-chain-go/config" ) const ( @@ -105,6 +106,14 @@ func IsConsensusBitmapValid( return ErrWrongSizeBitmap } + paddingBits := consensusSize % 8 + if paddingBits != 0 { + paddingMask := byte(0xFF << paddingBits) + if bitmap[len(bitmap)-1]&paddingMask != 0 { + return ErrPaddingBitsSet + } + } + numOfOnesInBitmap := 0 for index := range bitmap { numOfOnesInBitmap += bits.OnesCount8(bitmap[index]) diff --git a/common/errors.go b/common/errors.go index 3be75377813..2b5499d5d08 100644 --- a/common/errors.go +++ b/common/errors.go @@ -34,3 +34,6 @@ var ErrInvalidHashShardKey = errors.New("invalid hash shard key") // ErrInvalidNonceShardKey signals that the provided nonce-shard key is invalid var ErrInvalidNonceShardKey = errors.New("invalid nonce shard key") + +// ErrPaddingBitsSet signals that the provided bitmap has padding bits set to 1 instead of 0 +var ErrPaddingBitsSet = errors.New("padding bits in the bitmap should be zero") From d89e809d294bb82e46abb95c5e8fc4395d54876f Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 30 Apr 2026 16:01:17 +0300 Subject: [PATCH 012/116] fix #137 --- consensus/spos/worker.go | 10 +++++----- consensus/spos/worker_test.go | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 86cd3b39dfa..e8dbec25648 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -506,6 +506,11 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP wrk.consensusState.ResetRoundsWithoutReceivedMessages(cnsMsg.GetPubKey(), message.Peer()) + err = wrk.checkValidityAndProcessFinalInfo(cnsMsg, message) + if err != nil { + return nil, err + } + if wrk.nodeRedundancyHandler.IsRedundancyNode() { wrk.nodeRedundancyHandler.ResetInactivityIfNeeded( wrk.consensusState.SelfPubKey(), @@ -514,11 +519,6 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP ) } - err = wrk.checkValidityAndProcessFinalInfo(cnsMsg, message) - if err != nil { - return nil, err - } - wrk.networkShardingCollector.UpdatePeerIDInfo(message.Peer(), cnsMsg.PubKey, wrk.shardCoordinator.SelfId()) msgType := consensus.MessageType(cnsMsg.MsgType) diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index a144b88dcff..18bbc2c7ac4 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -607,10 +607,30 @@ func TestWorker_ProcessReceivedMessageRedundancyNodeShouldResetInactivityIfNeede }, } wrk.SetNodeRedundancyHandler(nodeRedundancyMock) - buff, _ := wrk.Marshalizer().Marshal(&consensus.Message{}) + hdr := &block.Header{ChainID: chainID} + hdrHash, _ := core.CalculateHash(mock.MarshalizerMock{}, &hashingMocks.HasherMock{}, hdr) + hdrStr, _ := mock.MarshalizerMock{}.Marshal(hdr) + cnsMsg := consensus.NewConsensusMessage( + hdrHash, + nil, + nil, + hdrStr, + []byte(wrk.ConsensusState().ConsensusGroup()[0]), + signature, + int(bls.MtBlockHeader), + 0, + chainID, + nil, + nil, + nil, + currentPid, + nil, + ) + buff, _ := wrk.Marshalizer().Marshal(cnsMsg) _, _ = wrk.ProcessReceivedMessage( &p2pmocks.P2PMessageMock{ DataField: buff, + PeerField: currentPid, SignatureField: []byte("signature"), }, fromConnectedPeerId, From f90753f1f2aa187dedf331949b907dafab87571a Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 30 Apr 2026 17:05:53 +0300 Subject: [PATCH 013/116] add try set busy --- common/disabled/processStatusHandler.go | 3 + common/disabled/processStatusHandler_test.go | 1 + common/interface.go | 1 + process/block/baseProcess.go | 4 +- process/block/baseProcess_test.go | 32 +++++++- process/block/metablock.go | 13 ++- process/block/metablock_test.go | 49 ++++++++++- process/block/shardblock.go | 13 ++- process/block/shardblock_test.go | 57 ++++++++++++- statusHandler/processStatusHandler.go | 16 ++++ statusHandler/processStatusHandler_test.go | 86 ++++++++++++++++++++ testscommon/processStatusHandlerStub.go | 16 +++- 12 files changed, 273 insertions(+), 18 deletions(-) diff --git a/common/disabled/processStatusHandler.go b/common/disabled/processStatusHandler.go index 036075099af..a6cd2f2bab4 100644 --- a/common/disabled/processStatusHandler.go +++ b/common/disabled/processStatusHandler.go @@ -13,6 +13,9 @@ func NewProcessStatusHandler() *processStatusHandler { // SetBusy does nothing func (psh *processStatusHandler) SetBusy(_ string) {} +// TrySetBusy returns true +func (psh *processStatusHandler) TrySetBusy(_ string) bool { return true } + // SetIdle does nothing func (psh *processStatusHandler) SetIdle() {} diff --git a/common/disabled/processStatusHandler_test.go b/common/disabled/processStatusHandler_test.go index a8860da4605..6bda4d4769e 100644 --- a/common/disabled/processStatusHandler_test.go +++ b/common/disabled/processStatusHandler_test.go @@ -21,6 +21,7 @@ func TestProcessStatusHandler_MethodsShouldNotPanic(t *testing.T) { psh := NewProcessStatusHandler() assert.False(t, check.IfNil(psh)) psh.SetBusy("") + assert.True(t, psh.TrySetBusy("")) psh.SetIdle() assert.True(t, psh.IsIdle()) } diff --git a/common/interface.go b/common/interface.go index c7f3059ff6f..0747b776f39 100644 --- a/common/interface.go +++ b/common/interface.go @@ -252,6 +252,7 @@ type StateStatisticsHandler interface { // able to tell if the node is idle or processing/committing a block type ProcessStatusHandler interface { SetBusy(reason string) + TrySetBusy(reason string) bool SetIdle() IsIdle() bool IsInterfaceNil() bool diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 982c667c78f..02a9bc0041c 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -2113,7 +2113,9 @@ func (bp *baseProcessor) Close() error { // ProcessScheduledBlock processes a scheduled block func (bp *baseProcessor) ProcessScheduledBlock(headerHandler data.HeaderHandler, bodyHandler data.BodyHandler, haveTime func() time.Duration) error { var err error - bp.processStatusHandler.SetBusy("baseProcessor.ProcessScheduledBlock") + if !bp.processStatusHandler.TrySetBusy("baseProcessor.ProcessScheduledBlock") { + return process.ErrBlockProcessorBusy + } defer func() { if err != nil { bp.RevertCurrentBlock() diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index 965cf1c04ad..90e205d79c7 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -2215,6 +2215,29 @@ func TestBaseProcessor_updateState(t *testing.T) { assert.Equal(t, []byte(strconv.Itoa(len(headers)-2)), cancelPruneRootHash) } +func TestBaseProcessor_ProcessScheduledBlockShouldErrWhenProcessorBusy(t *testing.T) { + t.Parallel() + + arguments := CreateMockArguments(createComponentHolderMocks()) + processHandler := arguments.CoreComponents.ProcessStatusHandler() + mockProcessHandler := processHandler.(*testscommon.ProcessStatusHandlerStub) + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { + return false + } + setIdleCalled := false + mockProcessHandler.SetIdleCalled = func() { + setIdleCalled = true + } + + bp, _ := blproc.NewShardProcessor(arguments) + + err := bp.ProcessScheduledBlock( + &block.MetaBlock{}, &block.Body{}, haveTime, + ) + require.Equal(t, process.ErrBlockProcessorBusy, err) + require.False(t, setIdleCalled, "SetIdle should not be called when TrySetBusy fails") +} + func TestBaseProcessor_ProcessScheduledBlockShouldFail(t *testing.T) { t.Parallel() @@ -2228,8 +2251,9 @@ func TestBaseProcessor_ProcessScheduledBlockShouldFail(t *testing.T) { mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } localErr := errors.New("execute all err") @@ -2259,8 +2283,9 @@ func TestBaseProcessor_ProcessScheduledBlockShouldFail(t *testing.T) { mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } localErr := errors.New("root hash err") @@ -2341,8 +2366,9 @@ func TestBaseProcessor_ProcessScheduledBlockShouldWork(t *testing.T) { mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } arguments.AccountsDB[state.UserAccountsState] = accounts diff --git a/process/block/metablock.go b/process/block/metablock.go index d0ef040abc0..b32527a8ad2 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -225,7 +225,9 @@ func (mp *metaProcessor) ProcessBlock( return process.ErrNilHaveTimeHandler } - mp.processStatusHandler.SetBusy("metaProcessor.ProcessBlock") + if !mp.processStatusHandler.TrySetBusy("metaProcessor.ProcessBlock") { + return process.ErrBlockProcessorBusy + } defer mp.processStatusHandler.SetIdle() err := mp.checkBlockValidity(headerHandler, bodyHandler) @@ -820,7 +822,9 @@ func (mp *metaProcessor) CreateBlock( return nil, nil, process.ErrWrongTypeAssertion } - mp.processStatusHandler.SetBusy("metaProcessor.CreateBlock") + if !mp.processStatusHandler.TrySetBusy("metaProcessor.CreateBlock") { + return nil, nil, process.ErrBlockProcessorBusy + } defer mp.processStatusHandler.SetIdle() metaHdr.SoftwareVersion = []byte(mp.headerIntegrityVerifier.GetVersion(metaHdr.Epoch)) @@ -1269,7 +1273,10 @@ func (mp *metaProcessor) CommitBlock( headerHandler data.HeaderHandler, bodyHandler data.BodyHandler, ) error { - mp.processStatusHandler.SetBusy("metaProcessor.CommitBlock") + if !mp.processStatusHandler.TrySetBusy("metaProcessor.CommitBlock") { + return process.ErrBlockProcessorBusy + } + var err error defer func() { if err != nil { diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index a0a663c5f66..1e96088936b 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -663,6 +663,49 @@ func TestMetaProcessor_ProcessBlockWithNilHaveTimeFuncShouldErr(t *testing.T) { assert.Equal(t, process.ErrNilHaveTimeHandler, err) } +func TestMetaProcessor_ProcessBlockShouldErrWhenProcessorBusy(t *testing.T) { + t.Parallel() + + arguments := createMockMetaArguments(createMockComponentHolders()) + + processHandler := arguments.CoreComponents.ProcessStatusHandler() + mockProcessHandler := processHandler.(*testscommon.ProcessStatusHandlerStub) + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { + return false + } + + mp, _ := blproc.NewMetaProcessor(arguments) + blk := &block.Body{} + + err := mp.ProcessBlock(&block.MetaBlock{ + Nonce: 1, + PubKeysBitmap: []byte("0100101"), + PrevHash: []byte(""), + Signature: []byte("signature"), + RootHash: []byte("roothash"), + }, blk, haveTime) + assert.Equal(t, process.ErrBlockProcessorBusy, err) +} + +func TestMetaProcessor_CreateBlockShouldErrWhenProcessorBusy(t *testing.T) { + t.Parallel() + + arguments := createMockMetaArguments(createMockComponentHolders()) + + processHandler := arguments.CoreComponents.ProcessStatusHandler() + mockProcessHandler := processHandler.(*testscommon.ProcessStatusHandlerStub) + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { + return false + } + + mp, _ := blproc.NewMetaProcessor(arguments) + + hdr, body, err := mp.CreateBlock(&block.MetaBlock{Round: 1}, func() bool { return true }) + assert.Equal(t, process.ErrBlockProcessorBusy, err) + assert.Nil(t, hdr) + assert.Nil(t, body) +} + func TestMetaProcessor_ProcessWithDirtyAccountShouldErr(t *testing.T) { t.Parallel() @@ -951,8 +994,9 @@ func TestMetaProcessor_CommitBlockStorageFailsForHeaderShouldNotReturnError(t *t mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } mp.SetHdrForCurrentBlock([]byte("hdr_hash1"), &block.Header{}, true) @@ -3075,8 +3119,9 @@ func TestMetaProcessor_CreateAndProcessBlockCallsProcessAfterFirstEpoch(t *testi mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } mp, _ := blproc.NewMetaProcessor(arguments) diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 8b93dc145f2..10485618b72 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -173,7 +173,9 @@ func (sp *shardProcessor) ProcessBlock( return process.ErrNilHaveTimeHandler } - sp.processStatusHandler.SetBusy("shardProcessor.ProcessBlock") + if !sp.processStatusHandler.TrySetBusy("shardProcessor.ProcessBlock") { + return process.ErrBlockProcessorBusy + } defer sp.processStatusHandler.SetIdle() err := sp.checkBlockValidity(headerHandler, bodyHandler) @@ -911,7 +913,9 @@ func (sp *shardProcessor) CreateBlock( return nil, nil, process.ErrWrongTypeAssertion } - sp.processStatusHandler.SetBusy("shardProcessor.CreateBlock") + if !sp.processStatusHandler.TrySetBusy("shardProcessor.CreateBlock") { + return nil, nil, process.ErrBlockProcessorBusy + } defer sp.processStatusHandler.SetIdle() err := sp.createBlockStarted() @@ -997,8 +1001,11 @@ func (sp *shardProcessor) CommitBlock( headerHandler data.HeaderHandler, bodyHandler data.BodyHandler, ) error { + if !sp.processStatusHandler.TrySetBusy("shardProcessor.CommitBlock") { + return process.ErrBlockProcessorBusy + } + var err error - sp.processStatusHandler.SetBusy("shardProcessor.CommitBlock") defer func() { if err != nil { sp.RevertCurrentBlock() diff --git a/process/block/shardblock_test.go b/process/block/shardblock_test.go index 24051d6f7b1..aebeaaf4bec 100644 --- a/process/block/shardblock_test.go +++ b/process/block/shardblock_test.go @@ -298,8 +298,9 @@ func TestShardProcess_CreateNewBlockHeaderProcessHeaderExpectCheckRoundCalled(t mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } err = shardProcessor.ProcessBlock(headerHandler, bodyHandler, func() time.Duration { return time.Second }) @@ -308,6 +309,54 @@ func TestShardProcess_CreateNewBlockHeaderProcessHeaderExpectCheckRoundCalled(t assert.Equal(t, []string{busyIdentifier, idleIdentifier}, busyIdleCalled) // the order is important } +func TestShardProcessor_ProcessBlockShouldErrWhenProcessorBusy(t *testing.T) { + t.Parallel() + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + + processHandler := arguments.CoreComponents.ProcessStatusHandler() + mockProcessHandler := processHandler.(*testscommon.ProcessStatusHandlerStub) + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { + return false + } + + sp, _ := blproc.NewShardProcessor(arguments) + header := &block.Header{ + Nonce: 1, + PubKeysBitmap: []byte("0100101"), + PrevHash: []byte(""), + PrevRandSeed: []byte("rand seed"), + Signature: []byte("signature"), + RootHash: []byte("roothash"), + } + body := &block.Body{} + + err := sp.ProcessBlock(header, body, func() time.Duration { return time.Second }) + require.Equal(t, process.ErrBlockProcessorBusy, err) +} + +func TestShardProcessor_CreateBlockShouldErrWhenProcessorBusy(t *testing.T) { + t.Parallel() + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + + processHandler := arguments.CoreComponents.ProcessStatusHandler() + mockProcessHandler := processHandler.(*testscommon.ProcessStatusHandlerStub) + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { + return false + } + + sp, _ := blproc.NewShardProcessor(arguments) + header := &block.Header{Round: 1} + + hdr, body, err := sp.CreateBlock(header, func() bool { return true }) + require.Equal(t, process.ErrBlockProcessorBusy, err) + require.Nil(t, hdr) + require.Nil(t, body) +} + func TestShardProcessor_ProcessWithDirtyAccountShouldErr(t *testing.T) { t.Parallel() // set accounts dirty @@ -1929,8 +1978,9 @@ func TestShardProcessor_CommitBlockStorageFailsForHeaderShouldErr(t *testing.T) mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } expectedFirstNonce := core.OptionalUint64{ HasValue: false, @@ -5355,8 +5405,9 @@ func TestShardProcessor_CreateBlock(t *testing.T) { mockProcessHandler.SetIdleCalled = func() { busyIdleCalled = append(busyIdleCalled, idleIdentifier) } - mockProcessHandler.SetBusyCalled = func(reason string) { + mockProcessHandler.TrySetBusyCalled = func(reason string) bool { busyIdleCalled = append(busyIdleCalled, busyIdentifier) + return true } expectedBusyIdleSequencePerCall := []string{busyIdentifier, idleIdentifier} diff --git a/statusHandler/processStatusHandler.go b/statusHandler/processStatusHandler.go index c54104f81a0..b18e6fff767 100644 --- a/statusHandler/processStatusHandler.go +++ b/statusHandler/processStatusHandler.go @@ -29,6 +29,22 @@ func (psh *processStatusHandler) SetBusy(reason string) { psh.mutStatus.Unlock() } +// TrySetBusy will atomically check if idle and set the internal state to "busy". +// Returns true if the state was successfully set to busy, false if already busy. +func (psh *processStatusHandler) TrySetBusy(reason string) bool { + psh.mutStatus.Lock() + defer psh.mutStatus.Unlock() + + if !psh.isIdle { + log.Debug("processStatusHandler.TrySetBusy: already busy", "reason", reason) + return false + } + + log.Debug("processStatusHandler.TrySetBusy", "reason", reason) + psh.isIdle = false + return true +} + // SetIdle will set the internal state to "idle" func (psh *processStatusHandler) SetIdle() { log.Debug("processStatusHandler.SetIdle") diff --git a/statusHandler/processStatusHandler_test.go b/statusHandler/processStatusHandler_test.go index e0443396dfb..8f863a5d4e8 100644 --- a/statusHandler/processStatusHandler_test.go +++ b/statusHandler/processStatusHandler_test.go @@ -2,6 +2,7 @@ package statusHandler import ( "sync" + "sync/atomic" "testing" "github.com/multiversx/mx-chain-core-go/core/check" @@ -34,6 +35,68 @@ func TestProcessStatusHandler_AllMethods(t *testing.T) { assert.True(t, psh.IsIdle()) } +func TestProcessStatusHandler_TrySetBusy(t *testing.T) { + t.Parallel() + + t.Run("should succeed when idle", func(t *testing.T) { + t.Parallel() + + psh := NewProcessStatusHandler() + assert.True(t, psh.IsIdle()) + + result := psh.TrySetBusy("reason") + assert.True(t, result) + assert.False(t, psh.IsIdle()) + }) + + t.Run("should fail when already busy", func(t *testing.T) { + t.Parallel() + + psh := NewProcessStatusHandler() + psh.SetBusy("first reason") + + result := psh.TrySetBusy("second reason") + assert.False(t, result) + assert.False(t, psh.IsIdle()) + }) + + t.Run("should succeed after SetIdle", func(t *testing.T) { + t.Parallel() + + psh := NewProcessStatusHandler() + psh.SetBusy("first reason") + psh.SetIdle() + + result := psh.TrySetBusy("second reason") + assert.True(t, result) + assert.False(t, psh.IsIdle()) + }) + + t.Run("second TrySetBusy should fail when first succeeded", func(t *testing.T) { + t.Parallel() + + psh := NewProcessStatusHandler() + + result1 := psh.TrySetBusy("first") + assert.True(t, result1) + + result2 := psh.TrySetBusy("second") + assert.False(t, result2) + assert.False(t, psh.IsIdle()) + }) + + t.Run("TrySetBusy should succeed again after SetIdle following a successful TrySetBusy", func(t *testing.T) { + t.Parallel() + + psh := NewProcessStatusHandler() + + assert.True(t, psh.TrySetBusy("first")) + psh.SetIdle() + assert.True(t, psh.TrySetBusy("second")) + assert.False(t, psh.IsIdle()) + }) +} + func TestNewProcessStatusHandler_ParallelCalls(t *testing.T) { t.Parallel() @@ -61,3 +124,26 @@ func TestNewProcessStatusHandler_ParallelCalls(t *testing.T) { wg.Wait() } + +func TestNewProcessStatusHandler_TrySetBusyConcurrency(t *testing.T) { + t.Parallel() + + psh := NewProcessStatusHandler() + numGoroutines := 100 + successCount := int32(0) + wg := sync.WaitGroup{} + wg.Add(numGoroutines) + + for i := 0; i < numGoroutines; i++ { + go func() { + defer wg.Done() + if psh.TrySetBusy("concurrent reason") { + atomic.AddInt32(&successCount, 1) + } + }() + } + + wg.Wait() + assert.Equal(t, int32(1), atomic.LoadInt32(&successCount)) + assert.False(t, psh.IsIdle()) +} diff --git a/testscommon/processStatusHandlerStub.go b/testscommon/processStatusHandlerStub.go index dcb39b90b18..c293b4f91b9 100644 --- a/testscommon/processStatusHandlerStub.go +++ b/testscommon/processStatusHandlerStub.go @@ -2,9 +2,10 @@ package testscommon // ProcessStatusHandlerStub - type ProcessStatusHandlerStub struct { - SetBusyCalled func(reason string) - SetIdleCalled func() - IsIdleCalled func() bool + SetBusyCalled func(reason string) + TrySetBusyCalled func(reason string) bool + SetIdleCalled func() + IsIdleCalled func() bool } // SetBusy - @@ -14,6 +15,15 @@ func (stub *ProcessStatusHandlerStub) SetBusy(reason string) { } } +// TrySetBusy - +func (stub *ProcessStatusHandlerStub) TrySetBusy(reason string) bool { + if stub.TrySetBusyCalled != nil { + return stub.TrySetBusyCalled(reason) + } + + return true +} + // SetIdle - func (stub *ProcessStatusHandlerStub) SetIdle() { if stub.SetIdleCalled != nil { From cfa6b18bc3b57415799d78f59c78fdb30bc2e4cb Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 30 Apr 2026 17:07:58 +0300 Subject: [PATCH 014/116] add try set busy --- process/errors.go | 3 +++ process/sync/baseSync.go | 7 +++++++ process/sync/shardblock_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/process/errors.go b/process/errors.go index ae4d7e9294f..ccf7857a449 100644 --- a/process/errors.go +++ b/process/errors.go @@ -131,6 +131,9 @@ var ErrValidatorStatsRootHashDoesNotMatch = errors.New("root hash for validator // ErrAccountStateDirty signals that the accounts were modified before starting the current modification var ErrAccountStateDirty = errors.New("accountState was dirty before starting to change") +// ErrBlockProcessorBusy signals that the block processor is already busy processing another block +var ErrBlockProcessorBusy = errors.New("block processor is busy") + // ErrInvalidShardId signals that the shard id is invalid var ErrInvalidShardId = errors.New("invalid shard id") diff --git a/process/sync/baseSync.go b/process/sync/baseSync.go index f66330346df..88836c4ce3d 100644 --- a/process/sync/baseSync.go +++ b/process/sync/baseSync.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "errors" "fmt" "math" "sync" @@ -669,6 +670,12 @@ func (boot *baseBootstrap) syncBlocks(ctx context.Context) { } func (boot *baseBootstrap) doJobOnSyncBlockFail(bodyHandler data.BodyHandler, headerHandler data.HeaderHandler, err error) { + if errors.Is(err, process.ErrBlockProcessorBusy) { + // block processor is busy with another call (e.g. consensus processing the same block); + // no processing started, nothing to track or roll back - just retry on next sync iteration + return + } + processBlockStarted := !check.IfNil(bodyHandler) && !check.IfNil(headerHandler) isProcessWithError := processBlockStarted && err != process.ErrTimeIsOut diff --git a/process/sync/shardblock_test.go b/process/sync/shardblock_test.go index cba152bca5c..535f96b0434 100644 --- a/process/sync/shardblock_test.go +++ b/process/sync/shardblock_test.go @@ -2004,6 +2004,36 @@ func TestShardBootstrap_RequestMiniBlocksFromHeaderWithNonceIfMissing(t *testing assert.True(t, requestDataWasCalled) } +func TestShardBootstrap_DoJobOnSyncBlockFailShouldSkipWhenBlockProcessorBusy(t *testing.T) { + t.Parallel() + + args := CreateShardBootstrapMockArguments() + + forkDetectorMock := &mock.ForkDetectorMock{ + ResetProbableHighestNonceCalled: func() { + require.Fail(t, "should not have called ResetProbableHighestNonce") + }, + } + args.ForkDetector = forkDetectorMock + args.ChainHandler = &testscommon.ChainHandlerStub{ + GetCurrentBlockHeaderCalled: func() data.HeaderHandler { + return &block.Header{Nonce: 1} + }, + GetGenesisHeaderCalled: func() data.HeaderHandler { + return &block.Header{} + }, + } + + bs, _ := sync.NewShardBootstrap(args) + + initialSyncErrors := bs.GetNumSyncedWithErrorsForNonce(2) + + bs.DoJobOnSyncBlockFail(&block.Body{}, &block.Header{Nonce: 2}, process.ErrBlockProcessorBusy) + + afterSyncErrors := bs.GetNumSyncedWithErrorsForNonce(2) + assert.Equal(t, initialSyncErrors, afterSyncErrors, "sync error counter should not be incremented for busy processor") +} + func TestShardBootstrap_DoJobOnSyncBlockFailShouldNotResetProbableHighestNonceWhenAreNotEnoughErrorsPerNonce(t *testing.T) { t.Parallel() From 3dea03d439c6127e2283bd5b30bdffa5c950540c Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 4 May 2026 11:57:56 +0300 Subject: [PATCH 015/116] do not update the timer if the index exists --- process/interceptors/processor/chunk/chunk.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/process/interceptors/processor/chunk/chunk.go b/process/interceptors/processor/chunk/chunk.go index 96e7b8742a4..4f183ec6515 100644 --- a/process/interceptors/processor/chunk/chunk.go +++ b/process/interceptors/processor/chunk/chunk.go @@ -33,10 +33,12 @@ func (c *chunk) Put(chunkIndex uint32, buff []byte) { return } - existing := c.data[chunkIndex] + existingData, ok := c.data[chunkIndex] + if !ok { + c.lastUpdated = time.Now() + } c.data[chunkIndex] = buff - c.size = c.size - len(existing) + len(buff) - c.lastUpdated = time.Now() + c.size = c.size - len(existingData) + len(buff) } // TryAssembleAllChunks will try to assemble the original payload by iterating all available chunks From 270033f4fc7ba5a2d6e785f302b2bda3a6d8c289 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 4 May 2026 15:05:41 +0300 Subject: [PATCH 016/116] UpdatePeerIDPublicKeyPair now gets the timestamp from payload --- consensus/spos/worker.go | 4 +-- .../disabled/disabledPeerShardMapper.go | 2 +- go.mod | 2 +- go.sum | 4 +-- integrationTests/interface.go | 2 +- .../mock/networkShardingCollectorMock.go | 2 +- integrationTests/mock/peerShardMapperStub.go | 6 ++--- .../interceptedPeerAuthentication.go | 22 +++++++++------- .../interceptedPeerAuthentication_test.go | 26 +++++++++++++++++++ .../peerAuthenticationInterceptorProcessor.go | 6 ++--- ...AuthenticationInterceptorProcessor_test.go | 4 +-- process/interface.go | 2 +- process/mock/peerShardMapperStub.go | 6 ++--- sharding/networksharding/peerShardMapper.go | 2 +- .../networksharding/peerShardMapper_test.go | 2 +- .../p2pmocks/networkShardingCollectorStub.go | 6 ++--- 16 files changed, 64 insertions(+), 34 deletions(-) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index e8dbec25648..5599b535f0f 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -504,13 +504,13 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP return nil, err } - wrk.consensusState.ResetRoundsWithoutReceivedMessages(cnsMsg.GetPubKey(), message.Peer()) - err = wrk.checkValidityAndProcessFinalInfo(cnsMsg, message) if err != nil { return nil, err } + wrk.consensusState.ResetRoundsWithoutReceivedMessages(cnsMsg.GetPubKey(), message.Peer()) + if wrk.nodeRedundancyHandler.IsRedundancyNode() { wrk.nodeRedundancyHandler.ResetInactivityIfNeeded( wrk.consensusState.SelfPubKey(), diff --git a/epochStart/bootstrap/disabled/disabledPeerShardMapper.go b/epochStart/bootstrap/disabled/disabledPeerShardMapper.go index c4695c00c09..a0b68087db3 100644 --- a/epochStart/bootstrap/disabled/disabledPeerShardMapper.go +++ b/epochStart/bootstrap/disabled/disabledPeerShardMapper.go @@ -17,7 +17,7 @@ func (p *peerShardMapper) GetLastKnownPeerID(_ []byte) (core.PeerID, bool) { } // UpdatePeerIDPublicKeyPair does nothing -func (p *peerShardMapper) UpdatePeerIDPublicKeyPair(_ core.PeerID, _ []byte) { +func (p *peerShardMapper) UpdatePeerIDPublicKeyPair(_ core.PeerID, _ []byte, _ int64) { } // PutPeerIdShardId does nothing diff --git a/go.mod b/go.mod index 85e957d1889..d5fcb8821d9 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.3.0 - github.com/multiversx/mx-chain-core-go v1.4.1 + github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a github.com/multiversx/mx-chain-crypto-go v1.3.0 github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 diff --git a/go.sum b/go.sum index 9caaf789bc5..252b93e81e5 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.1 h1:ljs53jpdjtCohpaqm2n/dvTGrFlSgIpoZYH8RVt5cWo= -github.com/multiversx/mx-chain-core-go v1.4.1/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a h1:UWVheMivOd2M7XEhZjlUGpI/V8skd8MfYv0JngpH98E= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= diff --git a/integrationTests/interface.go b/integrationTests/interface.go index 23504565a25..6c7c69103d1 100644 --- a/integrationTests/interface.go +++ b/integrationTests/interface.go @@ -49,7 +49,7 @@ type NodesCoordinatorFactory interface { // NetworkShardingUpdater defines the updating methods used by the network sharding component type NetworkShardingUpdater interface { GetPeerInfo(pid core.PeerID) core.P2PPeerInfo - UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdShardId(pid core.PeerID, shardID uint32) UpdatePeerIDInfo(pid core.PeerID, pk []byte, shardID uint32) PutPeerIdSubType(pid core.PeerID, peerSubType core.P2PPeerSubType) diff --git a/integrationTests/mock/networkShardingCollectorMock.go b/integrationTests/mock/networkShardingCollectorMock.go index cfd163e88ea..d3c8a17901a 100644 --- a/integrationTests/mock/networkShardingCollectorMock.go +++ b/integrationTests/mock/networkShardingCollectorMock.go @@ -33,7 +33,7 @@ func NewNetworkShardingCollectorMock() *networkShardingCollectorMock { } // UpdatePeerIDPublicKeyPair - -func (nscm *networkShardingCollectorMock) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (nscm *networkShardingCollectorMock) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, _ int64) { nscm.mutMaps.Lock() nscm.peerIdPkMap[pid] = pk nscm.pkPeerIdMap[string(pk)] = pid diff --git a/integrationTests/mock/peerShardMapperStub.go b/integrationTests/mock/peerShardMapperStub.go index b32a1045c7b..4f49be5a4f9 100644 --- a/integrationTests/mock/peerShardMapperStub.go +++ b/integrationTests/mock/peerShardMapperStub.go @@ -5,7 +5,7 @@ import "github.com/multiversx/mx-chain-core-go/core" // PeerShardMapperStub - type PeerShardMapperStub struct { GetLastKnownPeerIDCalled func(pk []byte) (core.PeerID, bool) - UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdShardIdCalled func(pid core.PeerID, shardID uint32) PutPeerIdSubTypeCalled func(pid core.PeerID, peerSubType core.P2PPeerSubType) UpdatePeerIDInfoCalled func(pid core.PeerID, pk []byte, shardID uint32) @@ -19,9 +19,9 @@ func (psms *PeerShardMapperStub) UpdatePeerIDInfo(pid core.PeerID, pk []byte, sh } // UpdatePeerIDPublicKeyPair - -func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { if psms.UpdatePeerIDPublicKeyPairCalled != nil { - psms.UpdatePeerIDPublicKeyPairCalled(pid, pk) + psms.UpdatePeerIDPublicKeyPairCalled(pid, pk, timestamp) } } diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 9a205c56dda..2af87dfdd4c 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -139,22 +139,26 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { if err != nil { return err } - } - // Early exit if mapping already exists - existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) - if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { - return process.ErrPeerAlreadyAuthenticated + // Early exit if mapping already exists + existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) + if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { + return process.ErrPeerAlreadyAuthenticated + } + + if existingInfo.AuthenticationTimestamp > ipa.payload.Timestamp { + return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthenticationTimestamp) + } } - // Verify payload signature - err = ipa.signaturesHandler.Verify(ipa.peerAuthentication.Payload, ipa.peerId, ipa.peerAuthentication.PayloadSignature) + // Verify payload + err = ipa.payloadValidator.ValidateTimestamp(ipa.payload.Timestamp) if err != nil { return err } - // Verify payload - err = ipa.payloadValidator.ValidateTimestamp(ipa.payload.Timestamp) + // Verify payload signature + err = ipa.signaturesHandler.Verify(ipa.peerAuthentication.Payload, ipa.peerId, ipa.peerAuthentication.PayloadSignature) if err != nil { return err } diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index a48c3d4c7fd..4569371f1ed 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -292,6 +292,32 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) }) + t.Run("peer already authenticated with newer timestamp should return error", func(t *testing.T) { + t.Parallel() + + providedPA := createDefaultInterceptedPeerAuthentication() + + arg := createMockInterceptedPeerAuthenticationArg(providedPA) + + authTimestamp := time.Now().Add(time.Minute).Unix() + arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ + VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { + require.Fail(t, "should have not been called") + return expectedErr + }, + } + arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + return core.P2PPeerInfo{ + AuthenticationTimestamp: authTimestamp, + } + }, + } + + ipa, _ := NewInterceptedPeerAuthentication(arg) + err := ipa.CheckValidity() + assert.ErrorIs(t, err, process.ErrPeerAlreadyAuthenticated) + }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 5864dcfcbf8..1c8950e2a8b 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -82,10 +82,10 @@ func (paip *peerAuthenticationInterceptorProcessor) Save(data process.Intercepte return err } - return paip.updatePeerInfo(interceptedPeerAuthenticationData.Message(), interceptedPeerAuthenticationData.SizeInBytes()) + return paip.updatePeerInfo(interceptedPeerAuthenticationData.Message(), interceptedPeerAuthenticationData.SizeInBytes(), payload.Timestamp) } -func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message interface{}, messageSize int) error { +func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message interface{}, messageSize int, payloadTimestamp int64) error { peerAuthenticationData, ok := message.(*heartbeat.PeerAuthentication) if !ok { return process.ErrWrongTypeAssertion @@ -93,7 +93,7 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter pidBytes := peerAuthenticationData.GetPid() paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey()) + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey(), payloadTimestamp) log.Trace("PeerAuthentication message saved") diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 09016fbc0af..d941cb79df1 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -136,7 +136,7 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { wasCalled := false args := createPeerAuthenticationInterceptorProcessArg() args.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ - UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { + UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte, timestamp int64) { wasCalled = true }, } @@ -205,7 +205,7 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { } wasUpdatePeerIDPublicKeyPairCalled := false arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ - UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { + UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte, timestamp int64) { wasUpdatePeerIDPublicKeyPairCalled = true assert.Equal(t, providedIPAMessage.Pid, pid.Bytes()) assert.Equal(t, providedIPAMessage.Pubkey, pk) diff --git a/process/interface.go b/process/interface.go index 99bafaa1354..eb0feedbf34 100644 --- a/process/interface.go +++ b/process/interface.go @@ -789,7 +789,7 @@ type PeerBlackListCacher interface { // PeerShardMapper can return the public key of a provided peer ID type PeerShardMapper interface { - UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdShardId(pid core.PeerID, shardID uint32) PutPeerIdSubType(pid core.PeerID, peerSubType core.P2PPeerSubType) GetPeerInfo(pid core.PeerID) core.P2PPeerInfo diff --git a/process/mock/peerShardMapperStub.go b/process/mock/peerShardMapperStub.go index 8c73a582904..364c19dd0ab 100644 --- a/process/mock/peerShardMapperStub.go +++ b/process/mock/peerShardMapperStub.go @@ -9,7 +9,7 @@ type PeerShardMapperStub struct { UpdatePeerIdPublicKeyCalled func(pid core.PeerID, pk []byte) UpdatePublicKeyShardIdCalled func(pk []byte, shardId uint32) PutPeerIdShardIdCalled func(pid core.PeerID, shardId uint32) - UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdSubTypeCalled func(pid core.PeerID, peerSubType core.P2PPeerSubType) } @@ -32,9 +32,9 @@ func (psms *PeerShardMapperStub) GetPeerInfo(pid core.PeerID) core.P2PPeerInfo { } // UpdatePeerIDPublicKeyPair - -func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { if psms.UpdatePeerIDPublicKeyPairCalled != nil { - psms.UpdatePeerIDPublicKeyPairCalled(pid, pk) + psms.UpdatePeerIDPublicKeyPairCalled(pid, pk, timestamp) } } diff --git a/sharding/networksharding/peerShardMapper.go b/sharding/networksharding/peerShardMapper.go index f66bbf52742..71def8d4598 100644 --- a/sharding/networksharding/peerShardMapper.go +++ b/sharding/networksharding/peerShardMapper.go @@ -231,7 +231,7 @@ func (psm *PeerShardMapper) getPeerInfoSearchingPidInFallbackCache(pid core.Peer // UpdatePeerIDPublicKeyPair updates the public key - peer ID pair in the corresponding maps // It also uses the intermediate pkPeerId cache that will prevent having thousands of peer ID's with // the same MultiversX PK that will make the node prone to an eclipse attack -func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, _ int64) { isNew := psm.updatePeerIDPublicKey(pid, pk) if isNew { peerLog.Trace("new peer mapping", "pid", pid.Pretty(), "pk", pk) diff --git a/sharding/networksharding/peerShardMapper_test.go b/sharding/networksharding/peerShardMapper_test.go index 6b03abe6805..17cdbd47464 100644 --- a/sharding/networksharding/peerShardMapper_test.go +++ b/sharding/networksharding/peerShardMapper_test.go @@ -231,7 +231,7 @@ func TestPeerShardMapper_UpdatePeerIDPublicKeyPairShouldWork(t *testing.T) { pid := core.PeerID("dummy peer ID") pk := []byte("dummy pk") - psm.UpdatePeerIDPublicKeyPair(pid, pk) + psm.UpdatePeerIDPublicKeyPair(pid, pk, 0) pkRecovered := psm.GetPkFromPidPk(pid) assert.Equal(t, pk, pkRecovered) diff --git a/testscommon/p2pmocks/networkShardingCollectorStub.go b/testscommon/p2pmocks/networkShardingCollectorStub.go index b7b1d3fb21b..bee8f438d46 100644 --- a/testscommon/p2pmocks/networkShardingCollectorStub.go +++ b/testscommon/p2pmocks/networkShardingCollectorStub.go @@ -6,7 +6,7 @@ import ( // NetworkShardingCollectorStub - type NetworkShardingCollectorStub struct { - UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte, timestamp int64) UpdatePeerIDInfoCalled func(pid core.PeerID, pk []byte, shardID uint32) PutPeerIdShardIdCalled func(pid core.PeerID, shardId uint32) PutPeerIdSubTypeCalled func(pid core.PeerID, peerSubType core.P2PPeerSubType) @@ -15,9 +15,9 @@ type NetworkShardingCollectorStub struct { } // UpdatePeerIDPublicKeyPair - -func (nscs *NetworkShardingCollectorStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (nscs *NetworkShardingCollectorStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { if nscs.UpdatePeerIDPublicKeyPairCalled != nil { - nscs.UpdatePeerIDPublicKeyPairCalled(pid, pk) + nscs.UpdatePeerIDPublicKeyPairCalled(pid, pk, timestamp) } } From cfb03106436489afb6fd6924aaa4fd6d1498d667 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Tue, 5 May 2026 11:35:31 +0300 Subject: [PATCH 017/116] finish implementation on psm --- go.mod | 2 +- go.sum | 4 +-- .../interceptedPeerAuthentication.go | 4 +-- .../interceptedPeerAuthentication_test.go | 2 +- sharding/networksharding/peerShardMapper.go | 33 ++++++++++++++++--- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index d5fcb8821d9..70cb5650b0a 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.3.0 - github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a + github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 github.com/multiversx/mx-chain-crypto-go v1.3.0 github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 diff --git a/go.sum b/go.sum index 252b93e81e5..f5a56769576 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a h1:UWVheMivOd2M7XEhZjlUGpI/V8skd8MfYv0JngpH98E= -github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 h1:hpnYOT5cDJip7B6GvFRSOcUdwdhBbuFsNjLvOxzYOn8= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 2af87dfdd4c..f1b43d294a9 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -146,8 +146,8 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { return process.ErrPeerAlreadyAuthenticated } - if existingInfo.AuthenticationTimestamp > ipa.payload.Timestamp { - return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthenticationTimestamp) + if existingInfo.AuthTimestamp > ipa.payload.Timestamp { + return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthTimestamp) } } diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 4569371f1ed..84ae30f5c48 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -309,7 +309,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { return core.P2PPeerInfo{ - AuthenticationTimestamp: authTimestamp, + AuthTimestamp: authTimestamp, } }, } diff --git a/sharding/networksharding/peerShardMapper.go b/sharding/networksharding/peerShardMapper.go index 71def8d4598..4e2fd1bfd5f 100644 --- a/sharding/networksharding/peerShardMapper.go +++ b/sharding/networksharding/peerShardMapper.go @@ -18,6 +18,7 @@ import ( const maxNumPidsPerPk = 3 const uint32Size = 4 +const int64Size = 8 const defaultShardId = uint32(0) const indexNotFound = -1 @@ -38,6 +39,7 @@ var _ p2p.PeerShardResolver = (*PeerShardMapper)(nil) type PeerShardMapper struct { peerIdPkCache storage.Cacher pkPeerIdCache storage.Cacher + pkTimestampCache storage.Cacher fallbackPkShardCache storage.Cacher fallbackPidShardCache storage.Cacher peerIdSubTypeCache storage.Cacher @@ -85,9 +87,15 @@ func NewPeerShardMapper(arg ArgPeerShardMapper) (*PeerShardMapper, error) { return nil, err } + pkTimestamp, err := cache.NewLRUCache(arg.PeerIdPkCache.MaxSize()) + if err != nil { + return nil, err + } + return &PeerShardMapper{ peerIdPkCache: arg.PeerIdPkCache, pkPeerIdCache: pkPeerId, + pkTimestampCache: pkTimestamp, fallbackPkShardCache: arg.FallbackPkShardCache, fallbackPidShardCache: arg.FallbackPidShardCache, peerIdSubTypeCache: peerIdSubTypeCache, @@ -161,12 +169,27 @@ func (psm *PeerShardMapper) getPeerInfoWithNodesCoordinator(pid core.PeerID) (*c } return &core.P2PPeerInfo{ - PeerType: core.ValidatorPeer, - ShardID: shardId, - PkBytes: pkBuff, + PeerType: core.ValidatorPeer, + ShardID: shardId, + PkBytes: pkBuff, + AuthTimestamp: psm.getTimestampForPk(pkBuff), }, true } +func (psm *PeerShardMapper) getTimestampForPk(pkBytes []byte) int64 { + timestamp, ok := psm.pkTimestampCache.Get(pkBytes) + if !ok { + return 0 + } + + timestampInt, ok := timestamp.(int64) + if !ok { + return 0 + } + + return timestampInt +} + func (psm *PeerShardMapper) getShardIDSearchingPkInFallbackCache(pkBuff []byte) (shardId uint32, ok bool) { if len(pkBuff) == 0 { return defaultShardId, false @@ -231,11 +254,13 @@ func (psm *PeerShardMapper) getPeerInfoSearchingPidInFallbackCache(pid core.Peer // UpdatePeerIDPublicKeyPair updates the public key - peer ID pair in the corresponding maps // It also uses the intermediate pkPeerId cache that will prevent having thousands of peer ID's with // the same MultiversX PK that will make the node prone to an eclipse attack -func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, _ int64) { +func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { isNew := psm.updatePeerIDPublicKey(pid, pk) if isNew { peerLog.Trace("new peer mapping", "pid", pid.Pretty(), "pk", pk) } + + psm.pkTimestampCache.Put(pk, timestamp, int64Size) } // UpdatePeerIDInfo updates the public keys and the shard ID for the peer ID in the corresponding maps From 267bb417fb4d32e3194c6f8484e51c242ecb4ccc Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 6 May 2026 12:36:22 +0300 Subject: [PATCH 018/116] extra miniblocks checks --- .../interceptedBlocks/interceptedMiniblock.go | 24 +- .../interceptedMiniblock_test.go | 4 +- process/common.go | 75 ++++++ process/common_test.go | 252 ++++++++++++++++++ process/coordinator/process.go | 37 +-- process/coordinator/process_test.go | 114 -------- 6 files changed, 332 insertions(+), 174 deletions(-) diff --git a/process/block/interceptedBlocks/interceptedMiniblock.go b/process/block/interceptedBlocks/interceptedMiniblock.go index a6569697b0a..7739a78010b 100644 --- a/process/block/interceptedBlocks/interceptedMiniblock.go +++ b/process/block/interceptedBlocks/interceptedMiniblock.go @@ -94,29 +94,7 @@ func (inMb *InterceptedMiniblock) IsForCurrentShard() bool { func (inMb *InterceptedMiniblock) integrity() error { miniblock := inMb.miniblock - receiverNotCurrentShard := miniblock.ReceiverShardID >= inMb.shardCoordinator.NumberOfShards() && - (miniblock.ReceiverShardID != core.MetachainShardId && miniblock.ReceiverShardID != core.AllShardId) - if receiverNotCurrentShard { - return process.ErrInvalidShardId - } - - senderNotCurrentShard := miniblock.SenderShardID >= inMb.shardCoordinator.NumberOfShards() && - miniblock.SenderShardID != core.MetachainShardId - if senderNotCurrentShard { - return process.ErrInvalidShardId - } - - for _, txHash := range miniblock.TxHashes { - if txHash == nil { - return process.ErrNilTxHash - } - } - - if len(miniblock.GetReserved()) > maxLenMiniBlockReservedField { - return process.ErrReservedFieldInvalid - } - - return nil + return process.CheckMiniBlock(miniblock, inMb.shardCoordinator) } // Type returns the type of this intercepted data diff --git a/process/block/interceptedBlocks/interceptedMiniblock_test.go b/process/block/interceptedBlocks/interceptedMiniblock_test.go index 46b489b259d..95875c589a3 100644 --- a/process/block/interceptedBlocks/interceptedMiniblock_test.go +++ b/process/block/interceptedBlocks/interceptedMiniblock_test.go @@ -86,7 +86,7 @@ func TestInterceptedMiniblock_InvalidReceiverShardIdShouldErr(t *testing.T) { err := inMb.CheckValidity() - assert.Equal(t, process.ErrInvalidShardId, err) + assert.ErrorIs(t, err, process.ErrInvalidShardId) } func TestInterceptedMiniblock_InvalidSenderShardIdShouldErr(t *testing.T) { @@ -103,7 +103,7 @@ func TestInterceptedMiniblock_InvalidSenderShardIdShouldErr(t *testing.T) { err := inMb.CheckValidity() - assert.Equal(t, process.ErrInvalidShardId, err) + assert.ErrorIs(t, err, process.ErrInvalidShardId) } func TestInterceptedMiniblock_ContainsNilHashShouldErr(t *testing.T) { diff --git a/process/common.go b/process/common.go index f89ca9ef21f..b1342c2148b 100644 --- a/process/common.go +++ b/process/common.go @@ -22,6 +22,7 @@ import ( vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" ) @@ -29,6 +30,7 @@ var log = logger.GetOrCreate("process") const maxSelfNotarizedLookback = 50 const VMStoragePrefix = "VM@" +const maxLenMiniBlockReservedField = 10 // ShardedCacheSearchMethod defines the algorithm for searching through a sharded cache type ShardedCacheSearchMethod byte @@ -1137,3 +1139,76 @@ func findSelfNotarizedMetaHeaderInBlock( return bestNonce, bestHeader, bestHash } + +// CheckMiniBlock will check miniblock validity +func CheckMiniBlock( + miniBlock *block.MiniBlock, + shardCoordinator sharding.Coordinator, +) error { + // shard id checks + receiverNotCurrentShard := miniBlock.ReceiverShardID >= shardCoordinator.NumberOfShards() && + (miniBlock.ReceiverShardID != core.MetachainShardId && miniBlock.ReceiverShardID != core.AllShardId) + if receiverNotCurrentShard { + return fmt.Errorf("%w - receiver not for current shard: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + miniBlock.Type, + miniBlock.SenderShardID, + miniBlock.ReceiverShardID) + } + + senderNotCurrentShard := miniBlock.SenderShardID >= shardCoordinator.NumberOfShards() && + miniBlock.SenderShardID != core.MetachainShardId + if senderNotCurrentShard { + return fmt.Errorf("%w - sender not for current shard: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + miniBlock.Type, + miniBlock.SenderShardID, + miniBlock.ReceiverShardID) + } + + if miniBlock.SenderShardID != shardCoordinator.SelfId() && miniBlock.GetReceiverShardID() != shardCoordinator.SelfId() && miniBlock.GetReceiverShardID() != core.AllShardId { + return fmt.Errorf("%w - not valid shard ids: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + miniBlock.Type, + miniBlock.SenderShardID, + miniBlock.ReceiverShardID) + } + + // type checks + if miniBlock.GetType() == block.PeerBlock && + (miniBlock.GetSenderShardID() != core.MetachainShardId || miniBlock.GetReceiverShardID() != core.AllShardId) { + return fmt.Errorf("%w - peer blocks: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + miniBlock.Type, + miniBlock.SenderShardID, + miniBlock.ReceiverShardID) + } + + if miniBlock.GetType() != block.PeerBlock && miniBlock.GetReceiverShardID() == core.AllShardId { + return fmt.Errorf("%w - invalid all shard ids: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + miniBlock.Type, + miniBlock.SenderShardID, + miniBlock.ReceiverShardID) + } + + if miniBlock.GetType() == block.RewardsBlock && miniBlock.GetSenderShardID() != core.MetachainShardId { + return fmt.Errorf("%w - invalid rewards block: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + miniBlock.Type, + miniBlock.SenderShardID, + miniBlock.ReceiverShardID) + } + + for _, txHash := range miniBlock.TxHashes { + if txHash == nil { + return ErrNilTxHash + } + } + + if len(miniBlock.GetReserved()) > maxLenMiniBlockReservedField { + return ErrReservedFieldInvalid + } + + return nil +} diff --git a/process/common_test.go b/process/common_test.go index b6e308ec3ab..c9dd3477938 100644 --- a/process/common_test.go +++ b/process/common_test.go @@ -2364,3 +2364,255 @@ func TestShardedCacheSearchMethod_ToString(t *testing.T) { str := process.ShardedCacheSearchMethod(166).ToString() assert.Equal(t, "unknown method 166", str) } + +func TestCheckMiniBlock(t *testing.T) { + t.Parallel() + + t.Run("not related to self shard, should fail", func(t *testing.T) { + t.Parallel() + + selfShardID := uint32(1) + shardCoordinator := &mock.ShardCoordinatorStub{ + SelfIdCalled: func() uint32 { + return selfShardID + }, + NumberOfShardsCalled: func() uint32 { + return 3 + }, + } + + mb := &block.MiniBlock{SenderShardID: 2, ReceiverShardID: 3, Type: block.TxBlock} + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: 2, ReceiverShardID: core.MetachainShardId, Type: block.TxBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: 3, Type: block.TxBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + }) + + t.Run("peer miniblock should be from meta to all shards", func(t *testing.T) { + t.Parallel() + + shardCoordinator := &mock.ShardCoordinatorStub{} + + mb := &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: core.AllShardId, Type: block.TxBlock} + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: 2, ReceiverShardID: core.AllShardId, Type: block.PeerBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: 1, Type: block.PeerBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: core.AllShardId, Type: block.PeerBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{SenderShardID: shardCoordinator.SelfId(), ReceiverShardID: core.MetachainShardId, Type: block.PeerBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + }) + + t.Run("rewards miniblock should be from meta", func(t *testing.T) { + t.Parallel() + + selfShardID := uint32(1) + + shardCoordinator := &mock.ShardCoordinatorStub{ + SelfIdCalled: func() uint32 { + return selfShardID + }, + NumberOfShardsCalled: func() uint32 { + return 3 + }, + } + + mb := &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: selfShardID, + Type: block.RewardsBlock, + } + err := process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: 2, + Type: block.RewardsBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: 0, + ReceiverShardID: 2, + Type: block.RewardsBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: 2, + ReceiverShardID: selfShardID, + Type: block.RewardsBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + }) + + t.Run("non peer miniblock should not be to all", func(t *testing.T) { + t.Parallel() + + shardCoordinator := &mock.ShardCoordinatorStub{} + + mb := &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: core.AllShardId, Type: block.TxBlock} + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.TxBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.ReceiptBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.RewardsBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.SmartContractResultBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + }) + + t.Run("wrong receiver shard id, should fail", func(t *testing.T) { + t.Parallel() + + wrongShardId := uint32(4) + shardCoordinator := &mock.ShardCoordinatorStub{ + NumberOfShardsCalled: func() uint32 { + return 2 + }, + } + + mb := &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: wrongShardId, Type: block.TxBlock} + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + }) + + t.Run("wrong sender shard id, should fail", func(t *testing.T) { + t.Parallel() + + wrongShardId := uint32(4) + shardCoordinator := &mock.ShardCoordinatorStub{ + NumberOfShardsCalled: func() uint32 { + return 2 + }, + } + + mb := &block.MiniBlock{SenderShardID: wrongShardId, ReceiverShardID: 1, Type: block.TxBlock} + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + }) + + t.Run("nil tx hash, should fail", func(t *testing.T) { + t.Parallel() + + shardCoordinator := &mock.ShardCoordinatorStub{ + NumberOfShardsCalled: func() uint32 { + return 3 + }, + } + + mb := &block.MiniBlock{ + SenderShardID: shardCoordinator.SelfId(), ReceiverShardID: 1, + Type: block.TxBlock, + TxHashes: [][]byte{[]byte("txHash0"), nil, []byte("txHash1")}, + } + + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrNilTxHash) + }) + + t.Run("invalid reserved field, should fail", func(t *testing.T) { + t.Parallel() + + shardCoordinator := &mock.ShardCoordinatorStub{ + NumberOfShardsCalled: func() uint32 { + return 3 + }, + } + + mb := &block.MiniBlock{ + SenderShardID: shardCoordinator.SelfId(), ReceiverShardID: 1, + Type: block.TxBlock, + Reserved: bytes.Repeat([]byte("A"), 100), + } + + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrReservedFieldInvalid) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() + + selfShardID := uint32(1) + + shardCoordinator := &mock.ShardCoordinatorStub{ + SelfIdCalled: func() uint32 { + return selfShardID + }, + NumberOfShardsCalled: func() uint32 { + return 3 + }, + } + + mb := &block.MiniBlock{ + SenderShardID: 2, + ReceiverShardID: selfShardID, + Type: block.TxBlock, + } + err := process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: 2, + Type: block.TxBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: selfShardID, + Type: block.TxBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: core.AllShardId, + Type: block.PeerBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: selfShardID, + Type: block.RewardsBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + }) +} diff --git a/process/coordinator/process.go b/process/coordinator/process.go index 7c18f53efcd..8670a3275f5 100644 --- a/process/coordinator/process.go +++ b/process/coordinator/process.go @@ -480,7 +480,7 @@ func (tc *transactionCoordinator) processMiniBlocksFromMe( haveTime func() bool, ) error { for _, mb := range body.MiniBlocks { - err := tc.checkMiniBlock(mb) + err := process.CheckMiniBlock(mb, tc.shardCoordinator) if err != nil { return err } @@ -524,39 +524,6 @@ func (tc *transactionCoordinator) processMiniBlocksFromMe( return nil } -func (tc *transactionCoordinator) checkMiniBlock( - miniBlock *block.MiniBlock, -) error { - // there are checks for non existing shard id at interceptors level - - if miniBlock.SenderShardID != tc.shardCoordinator.SelfId() && miniBlock.GetReceiverShardID() != tc.shardCoordinator.SelfId() && miniBlock.GetReceiverShardID() != core.AllShardId { - return fmt.Errorf("%w - not valid shard ids: block type: %s, sender shard id: %d, receiver shard id: %d", - process.ErrInvalidShardId, - miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) - } - - if miniBlock.GetType() == block.PeerBlock && - (miniBlock.GetSenderShardID() != core.MetachainShardId || miniBlock.GetReceiverShardID() != core.AllShardId) { - return fmt.Errorf("%w - peer blocks: block type: %s, sender shard id: %d, receiver shard id: %d", - process.ErrInvalidShardId, - miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) - } - - if miniBlock.GetType() != block.PeerBlock && miniBlock.GetReceiverShardID() == core.AllShardId { - return fmt.Errorf("%w - invalid all shard ids: block type: %s, sender shard id: %d, receiver shard id: %d", - process.ErrInvalidShardId, - miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) - } - - return nil -} - func (tc *transactionCoordinator) processMiniBlocksToMe( header data.HeaderHandler, body *block.Body, @@ -579,7 +546,7 @@ func (tc *transactionCoordinator) processMiniBlocksToMe( for mbIndex = 0; mbIndex < len(body.MiniBlocks); mbIndex++ { miniBlock := body.MiniBlocks[mbIndex] - err := tc.checkMiniBlock(miniBlock) + err := process.CheckMiniBlock(miniBlock, tc.shardCoordinator) if err != nil { return mbIndex, err } diff --git a/process/coordinator/process_test.go b/process/coordinator/process_test.go index f18f746ca5d..e57c6de1d50 100644 --- a/process/coordinator/process_test.go +++ b/process/coordinator/process_test.go @@ -4686,117 +4686,3 @@ func TestTransactionCoordinator_requestMissingMiniBlocksAndTransactionsShouldWor assert.Equal(t, 2, numTxsRequested) mutMap.RUnlock() } - -func TestTransactionCoordinator_checkMiniBlock(t *testing.T) { - t.Parallel() - - t.Run("valid miniblock should not error", func(t *testing.T) { - t.Parallel() - - argsTransactionCoordinator := createMockTransactionCoordinatorArguments() - - selfShardID := uint32(1) - argsTransactionCoordinator.ShardCoordinator = &mock.ShardCoordinatorStub{ - SelfIdCalled: func() uint32 { - return selfShardID - }, - } - - tc, err := NewTransactionCoordinator(argsTransactionCoordinator) - require.Nil(t, err) - require.NotNil(t, tc) - - mb := &block.MiniBlock{SenderShardID: 2, ReceiverShardID: selfShardID, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.Nil(t, err) - - mb = &block.MiniBlock{SenderShardID: selfShardID, ReceiverShardID: 2, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.Nil(t, err) - }) - - t.Run("not related to self shard, should fail", func(t *testing.T) { - t.Parallel() - - argsTransactionCoordinator := createMockTransactionCoordinatorArguments() - - selfShardID := uint32(1) - argsTransactionCoordinator.ShardCoordinator = &mock.ShardCoordinatorStub{ - SelfIdCalled: func() uint32 { - return selfShardID - }, - } - - tc, err := NewTransactionCoordinator(argsTransactionCoordinator) - require.Nil(t, err) - require.NotNil(t, tc) - - mb := &block.MiniBlock{SenderShardID: 2, ReceiverShardID: 3, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: 2, ReceiverShardID: core.MetachainShardId, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: 3, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - }) - - t.Run("peer miniblock should be from meta to all shards", func(t *testing.T) { - t.Parallel() - - argsTransactionCoordinator := createMockTransactionCoordinatorArguments() - - tc, err := NewTransactionCoordinator(argsTransactionCoordinator) - require.Nil(t, err) - require.NotNil(t, tc) - - mb := &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: core.AllShardId, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: 2, ReceiverShardID: core.AllShardId, Type: block.PeerBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: 1, Type: block.PeerBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: core.AllShardId, Type: block.PeerBlock} - err = tc.checkMiniBlock(mb) - require.Nil(t, err) - }) - - t.Run("non peer miniblock should not be to all", func(t *testing.T) { - t.Parallel() - - argsTransactionCoordinator := createMockTransactionCoordinatorArguments() - - tc, err := NewTransactionCoordinator(argsTransactionCoordinator) - require.Nil(t, err) - require.NotNil(t, tc) - - mb := &block.MiniBlock{SenderShardID: core.MetachainShardId, ReceiverShardID: core.AllShardId, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.TxBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.ReceiptBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.RewardsBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - - mb = &block.MiniBlock{SenderShardID: 1, ReceiverShardID: core.AllShardId, Type: block.SmartContractResultBlock} - err = tc.checkMiniBlock(mb) - require.ErrorIs(t, err, process.ErrInvalidShardId) - }) -} From 8914f4ce7b1b5255c8f4c60c0678672143c1b710 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 23 Apr 2026 12:33:40 +0300 Subject: [PATCH 019/116] early exit on peer authentication messages, but keep broadcast them further --- integrationTests/testHeartbeatNode.go | 1 + .../metaInterceptorsContainerFactory.go | 1 + .../shardInterceptorsContainerFactory.go | 1 + .../interceptedPeerAuthentication.go | 12 +++++++ .../interceptedPeerAuthentication_test.go | 36 +++++++++++++++++++ .../factory/argInterceptedDataFactory.go | 1 + .../interceptedMetaHeaderDataFactory_test.go | 1 + ...nterceptedPeerAuthenticationDataFactory.go | 3 ++ .../peerAuthenticationInterceptorProcessor.go | 10 +++++- ...AuthenticationInterceptorProcessor_test.go | 35 ++++++++++++++++++ 10 files changed, 100 insertions(+), 1 deletion(-) diff --git a/integrationTests/testHeartbeatNode.go b/integrationTests/testHeartbeatNode.go index 2b0f5ab28de..c407b8534a1 100644 --- a/integrationTests/testHeartbeatNode.go +++ b/integrationTests/testHeartbeatNode.go @@ -646,6 +646,7 @@ func (thn *TestHeartbeatNode) initInterceptors() { SignaturesHandler: &processMock.SignaturesHandlerStub{}, HeartbeatExpiryTimespanInSec: thn.heartbeatExpiryTimespanInSec, PeerID: thn.MainMessenger.ID(), + PeerShardMapper: thn.MainPeerShardMapper, } thn.createPeerAuthInterceptor(argsFactory) diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 40abe25221b..b3238a47cae 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -103,6 +103,7 @@ func NewMetaInterceptorsContainerFactory( SignaturesHandler: args.SignaturesHandler, HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, PeerID: args.MainMessenger.ID(), + PeerShardMapper: args.MainPeerShardMapper, } base := &baseInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index d144113d30f..7903041de61 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -104,6 +104,7 @@ func NewShardInterceptorsContainerFactory( SignaturesHandler: args.SignaturesHandler, HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, PeerID: args.MainMessenger.ID(), + PeerShardMapper: args.MainPeerShardMapper, } base := &baseInterceptorsContainerFactory{ diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index a10e5e6dd8d..8db1ca8f0cd 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -21,6 +21,7 @@ type ArgInterceptedPeerAuthentication struct { PeerSignatureHandler crypto.PeerSignatureHandler PayloadValidator process.PeerAuthenticationPayloadValidator HardforkTriggerPubKey []byte + PeerShardMapper process.PeerShardMapper } // interceptedPeerAuthentication is a wrapper over PeerAuthentication @@ -33,6 +34,7 @@ type interceptedPeerAuthentication struct { peerSignatureHandler crypto.PeerSignatureHandler payloadValidator process.PeerAuthenticationPayloadValidator hardforkTriggerPubKey []byte + peerShardMapper process.PeerShardMapper } // NewInterceptedPeerAuthentication tries to create a new intercepted peer authentication instance @@ -55,6 +57,7 @@ func NewInterceptedPeerAuthentication(arg ArgInterceptedPeerAuthentication) (*in peerSignatureHandler: arg.PeerSignatureHandler, payloadValidator: arg.PayloadValidator, hardforkTriggerPubKey: arg.HardforkTriggerPubKey, + peerShardMapper: arg.PeerShardMapper, } intercepted.peerId = core.PeerID(intercepted.peerAuthentication.Pid) @@ -81,6 +84,9 @@ func checkArg(arg ArgInterceptedPeerAuthentication) error { if len(arg.HardforkTriggerPubKey) == 0 { return fmt.Errorf("%w hardfork trigger public key bytes length is 0", process.ErrInvalidValue) } + if check.IfNil(arg.PeerShardMapper) { + return process.ErrNilPeerShardMapper + } return nil } @@ -135,6 +141,12 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { } } + // Early exit if mapping already exists + existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) + if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { + return nil + } + // Verify payload signature err = ipa.signaturesHandler.Verify(ipa.peerAuthentication.Payload, ipa.peerId, ipa.peerAuthentication.PayloadSignature) if err != nil { diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index fdc19e6a130..b16d2c85fbb 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var expectedErr = errors.New("expected error") @@ -59,6 +60,7 @@ func createMockInterceptedPeerAuthenticationArg(interceptedData *heartbeat.PeerA PeerSignatureHandler: &cryptoMocks.PeerSignatureHandlerStub{}, PayloadValidator: &testscommon.PeerAuthenticationPayloadValidatorStub{}, HardforkTriggerPubKey: providedHardforkPubKey, + PeerShardMapper: &processMocks.PeerShardMapperStub{}, } arg.DataBuff, _ = arg.Marshaller.Marshal(interceptedData) @@ -128,6 +130,16 @@ func TestNewInterceptedPeerAuthentication(t *testing.T) { assert.True(t, check.IfNil(ipa)) assert.Equal(t, process.ErrNilPeerSignatureHandler, err) }) + t.Run("nil peer shard mapper should error", func(t *testing.T) { + t.Parallel() + + arg := createMockInterceptedPeerAuthenticationArg(createDefaultInterceptedPeerAuthentication()) + arg.PeerShardMapper = nil + + ipa, err := NewInterceptedPeerAuthentication(arg) + assert.True(t, check.IfNil(ipa)) + assert.Equal(t, process.ErrNilPeerShardMapper, err) + }) t.Run("unmarshal returns error", func(t *testing.T) { t.Parallel() @@ -256,6 +268,30 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err = ipa.CheckValidity() assert.True(t, errors.Is(err, expectedErr)) }) + t.Run("peer already authenticated with same pubkey should early exit", func(t *testing.T) { + t.Parallel() + + providedPA := createDefaultInterceptedPeerAuthentication() + arg := createMockInterceptedPeerAuthenticationArg(providedPA) + + arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ + VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { + require.Fail(t, "should have not been called") + return expectedErr + }, + } + arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + return core.P2PPeerInfo{ + PkBytes: providedPA.Pubkey, + } + }, + } + + ipa, _ := NewInterceptedPeerAuthentication(arg) + err := ipa.CheckValidity() + assert.Nil(t, err) + }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/factory/argInterceptedDataFactory.go b/process/interceptors/factory/argInterceptedDataFactory.go index dbc7350436d..cb6d263e2b7 100644 --- a/process/interceptors/factory/argInterceptedDataFactory.go +++ b/process/interceptors/factory/argInterceptedDataFactory.go @@ -60,4 +60,5 @@ type ArgInterceptedDataFactory struct { SignaturesHandler process.SignaturesHandler HeartbeatExpiryTimespanInSec int64 PeerID core.PeerID + PeerShardMapper process.PeerShardMapper } diff --git a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go index f962fb9806e..6890990296b 100644 --- a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go +++ b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go @@ -138,6 +138,7 @@ func createMockArgument( SignaturesHandler: &processMocks.SignaturesHandlerStub{}, HeartbeatExpiryTimespanInSec: 30, PeerID: "pid", + PeerShardMapper: &processMocks.PeerShardMapperStub{}, } } diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index 18b4a4f40a2..a425dc3233a 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -21,6 +21,7 @@ type interceptedPeerAuthenticationDataFactory struct { peerSignatureHandler crypto.PeerSignatureHandler hardforkTriggerPubKey []byte payloadValidator process.PeerAuthenticationPayloadValidator + peerShardMapper process.PeerShardMapper } // NewInterceptedPeerAuthenticationDataFactory creates an instance of interceptedPeerAuthenticationDataFactory @@ -42,6 +43,7 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) peerSignatureHandler: arg.PeerSignatureHandler, payloadValidator: payloadValidator, hardforkTriggerPubKey: arg.CoreComponents.HardforkTriggerPubKey(), + peerShardMapper: arg.PeerShardMapper, }, nil } @@ -83,6 +85,7 @@ func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ cor PeerSignatureHandler: ipadf.peerSignatureHandler, PayloadValidator: ipadf.payloadValidator, HardforkTriggerPubKey: ipadf.hardforkTriggerPubKey, + PeerShardMapper: ipadf.peerShardMapper, } return heartbeat.NewInterceptedPeerAuthentication(arg) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 5864dcfcbf8..4633ee5aa7e 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -92,8 +92,16 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter } pidBytes := peerAuthenticationData.GetPid() + pid := core.PeerID(pidBytes) + + // early exit if info already saved + existingInfo := paip.peerShardMapper.GetPeerInfo(pid) + if string(existingInfo.PkBytes) == string(peerAuthenticationData.GetPubkey()) { + return nil + } + paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey()) + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) log.Trace("PeerAuthentication message saved") diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 3a1db0b6b66..2069c8d0d32 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" heartbeatMessages "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" @@ -63,6 +64,7 @@ func createMockInterceptedPeerAuthentication() process.InterceptedData { PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, PayloadValidator: payloadValidator, HardforkTriggerPubKey: []byte("provided hardfork pub key"), + PeerShardMapper: &mock.PeerShardMapperStub{}, } arg.DataBuff, _ = arg.Marshaller.Marshal(createInterceptedPeerAuthentication()) ipa, _ := heartbeat.NewInterceptedPeerAuthentication(arg) @@ -181,6 +183,39 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { err = paip.Save(createMockInterceptedPeerAuthentication(), "", "") assert.Equal(t, expectedError, err) }) + t.Run("peer info already saved should early exit", func(t *testing.T) { + t.Parallel() + + providedIPA := createMockInterceptedPeerAuthentication() + providedIPAHandler := providedIPA.(interceptedDataHandler) + providedIPAMessage := providedIPAHandler.Message().(*heartbeatMessages.PeerAuthentication) + + arg := createPeerAuthenticationInterceptorProcessArg() + arg.PeerAuthenticationCacher = &cache.CacherStub{ + PutCalled: func(key []byte, value interface{}, sizeInBytes int) (evicted bool) { + require.Fail(t, "should have not been called") + return false + }, + } + wasGetPeerInfoCalled := false + arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + wasGetPeerInfoCalled = true + assert.Equal(t, providedIPAMessage.Pid, pid.Bytes()) + return core.P2PPeerInfo{ + PkBytes: providedIPAMessage.Pubkey, + } + }, + } + + paip, err := processor.NewPeerAuthenticationInterceptorProcessor(arg) + assert.Nil(t, err) + assert.False(t, paip.IsInterfaceNil()) + + err = paip.Save(providedIPA, "", "") + assert.Nil(t, err) + assert.True(t, wasGetPeerInfoCalled) + }) t.Run("should work", func(t *testing.T) { t.Parallel() From f1f1d01eefe42f06c5b9c71ff438d7706d45d36b Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 23 Apr 2026 13:01:44 +0300 Subject: [PATCH 020/116] save the messages even though map already exists --- .../peerAuthenticationInterceptorProcessor.go | 9 +++------ .../peerAuthenticationInterceptorProcessor_test.go | 14 +++++--------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 4633ee5aa7e..1718756a806 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -94,15 +94,12 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter pidBytes := peerAuthenticationData.GetPid() pid := core.PeerID(pidBytes) - // early exit if info already saved + paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) existingInfo := paip.peerShardMapper.GetPeerInfo(pid) - if string(existingInfo.PkBytes) == string(peerAuthenticationData.GetPubkey()) { - return nil + if string(existingInfo.PkBytes) != string(peerAuthenticationData.GetPubkey()) { + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) } - paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) - log.Trace("PeerAuthentication message saved") return nil diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 2069c8d0d32..1aa2da79b26 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -6,9 +6,6 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - heartbeatMessages "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/heartbeat" @@ -19,6 +16,8 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type interceptedDataHandler interface { @@ -191,12 +190,6 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { providedIPAMessage := providedIPAHandler.Message().(*heartbeatMessages.PeerAuthentication) arg := createPeerAuthenticationInterceptorProcessArg() - arg.PeerAuthenticationCacher = &cache.CacherStub{ - PutCalled: func(key []byte, value interface{}, sizeInBytes int) (evicted bool) { - require.Fail(t, "should have not been called") - return false - }, - } wasGetPeerInfoCalled := false arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { @@ -206,6 +199,9 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { PkBytes: providedIPAMessage.Pubkey, } }, + UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { + require.Fail(t, "should not have been called") + }, } paip, err := processor.NewPeerAuthenticationInterceptorProcessor(arg) From ab36a857718bc9e9a09692c6c149000c4b403663 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 6 May 2026 14:04:47 +0300 Subject: [PATCH 021/116] return error when peer already authenticated # Conflicts: # process/errors.go --- process/errors.go | 3 ++ .../interceptedPeerAuthentication.go | 2 +- .../interceptedPeerAuthentication_test.go | 4 +-- .../peerAuthenticationInterceptorProcessor.go | 7 +---- ...AuthenticationInterceptorProcessor_test.go | 31 ------------------- 5 files changed, 7 insertions(+), 40 deletions(-) diff --git a/process/errors.go b/process/errors.go index dabdef5f176..958540cf177 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1322,3 +1322,6 @@ var ErrDuplicatedHashInBlock = errors.New("duplicated hash in block") // ErrDoubleTransactionsFound signals that double transactions found var ErrDoubleTransactionsFound = errors.New("double transactions found") + +// ErrPeerAlreadyAuthenticated signals that a peer authentication message was received for a peer that already has an existing mapping +var ErrPeerAlreadyAuthenticated = errors.New("peer already authenticated") diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 8db1ca8f0cd..9a205c56dda 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -144,7 +144,7 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { // Early exit if mapping already exists existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { - return nil + return process.ErrPeerAlreadyAuthenticated } // Verify payload signature diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index b16d2c85fbb..a48c3d4c7fd 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -268,7 +268,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err = ipa.CheckValidity() assert.True(t, errors.Is(err, expectedErr)) }) - t.Run("peer already authenticated with same pubkey should early exit", func(t *testing.T) { + t.Run("peer already authenticated with same pubkey should return error", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() @@ -290,7 +290,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { ipa, _ := NewInterceptedPeerAuthentication(arg) err := ipa.CheckValidity() - assert.Nil(t, err) + assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 1718756a806..5864dcfcbf8 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -92,13 +92,8 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter } pidBytes := peerAuthenticationData.GetPid() - pid := core.PeerID(pidBytes) - paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - existingInfo := paip.peerShardMapper.GetPeerInfo(pid) - if string(existingInfo.PkBytes) != string(peerAuthenticationData.GetPubkey()) { - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(pid, peerAuthenticationData.GetPubkey()) - } + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey()) log.Trace("PeerAuthentication message saved") diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 1aa2da79b26..09016fbc0af 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -17,7 +17,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) type interceptedDataHandler interface { @@ -182,36 +181,6 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { err = paip.Save(createMockInterceptedPeerAuthentication(), "", "") assert.Equal(t, expectedError, err) }) - t.Run("peer info already saved should early exit", func(t *testing.T) { - t.Parallel() - - providedIPA := createMockInterceptedPeerAuthentication() - providedIPAHandler := providedIPA.(interceptedDataHandler) - providedIPAMessage := providedIPAHandler.Message().(*heartbeatMessages.PeerAuthentication) - - arg := createPeerAuthenticationInterceptorProcessArg() - wasGetPeerInfoCalled := false - arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ - GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { - wasGetPeerInfoCalled = true - assert.Equal(t, providedIPAMessage.Pid, pid.Bytes()) - return core.P2PPeerInfo{ - PkBytes: providedIPAMessage.Pubkey, - } - }, - UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { - require.Fail(t, "should not have been called") - }, - } - - paip, err := processor.NewPeerAuthenticationInterceptorProcessor(arg) - assert.Nil(t, err) - assert.False(t, paip.IsInterfaceNil()) - - err = paip.Save(providedIPA, "", "") - assert.Nil(t, err) - assert.True(t, wasGetPeerInfoCalled) - }) t.Run("should work", func(t *testing.T) { t.Parallel() From 32317e950125f5f7169b0333d7336af5d375d6f4 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 24 Apr 2026 16:09:17 +0300 Subject: [PATCH 022/116] latest vm common from master --- factory/api/apiResolverFactory.go | 5 +++-- factory/processing/txSimulatorProcessComponents.go | 6 ++++-- go.mod | 4 ++-- go.sum | 8 ++++---- integrationTests/testProcessorNodeWithTestWebServer.go | 6 ++++-- integrationTests/vm/testInitializer.go | 5 +++-- node/external/transactionAPI/apiTransactionResults.go | 6 +++--- node/external/transactionAPI/interface.go | 2 +- node/external/transactionAPI/unmarshaller.go | 2 +- outport/process/transactionsfee/interface.go | 2 +- outport/process/transactionsfee/transactionChecker.go | 4 ++-- .../process/transactionsfee/transactionsFeeProcessor.go | 9 +++++---- process/transactionEvaluator/interface.go | 2 +- process/transactionEvaluator/transactionSimulator.go | 4 +++- testscommon/dataFieldParserStub.go | 2 +- 15 files changed, 38 insertions(+), 29 deletions(-) diff --git a/factory/api/apiResolverFactory.go b/factory/api/apiResolverFactory.go index feb1b1a4d24..e9f435853a6 100644 --- a/factory/api/apiResolverFactory.go +++ b/factory/api/apiResolverFactory.go @@ -222,8 +222,9 @@ func CreateApiResolver(args *ApiResolverArgs) (facade.ApiResolver, error) { } argsDataFieldParser := &datafield.ArgsOperationDataFieldParser{ - AddressLength: args.CoreComponents.AddressPubKeyConverter().Len(), - Marshalizer: args.CoreComponents.InternalMarshalizer(), + AddressLength: args.CoreComponents.AddressPubKeyConverter().Len(), + Marshalizer: args.CoreComponents.InternalMarshalizer(), + RelayedTransactionsV1V2DisableEpoch: args.CoreComponents.EnableEpochsHandler().GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), } dataFieldParser, err := datafield.NewOperationDataFieldParser(argsDataFieldParser) if err != nil { diff --git a/factory/processing/txSimulatorProcessComponents.go b/factory/processing/txSimulatorProcessComponents.go index 3b4878977e9..dcea258d65f 100644 --- a/factory/processing/txSimulatorProcessComponents.go +++ b/factory/processing/txSimulatorProcessComponents.go @@ -3,6 +3,7 @@ package processing import ( "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/disabled" bootstrapDisabled "github.com/multiversx/mx-chain-go/epochStart/bootstrap/disabled" "github.com/multiversx/mx-chain-go/factory" @@ -54,8 +55,9 @@ func (pcf *processComponentsFactory) createAPITransactionEvaluator(epochStartTri } dataFieldParser, err := datafield.NewOperationDataFieldParser(&datafield.ArgsOperationDataFieldParser{ - AddressLength: pcf.coreData.AddressPubKeyConverter().Len(), - Marshalizer: pcf.coreData.InternalMarshalizer(), + AddressLength: pcf.coreData.AddressPubKeyConverter().Len(), + Marshalizer: pcf.coreData.InternalMarshalizer(), + RelayedTransactionsV1V2DisableEpoch: pcf.coreData.EnableEpochsHandler().GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), }) if err != nil { return nil, nil, err diff --git a/go.mod b/go.mod index 3113541dea9..85e957d1889 100644 --- a/go.mod +++ b/go.mod @@ -19,11 +19,11 @@ require ( github.com/multiversx/mx-chain-communication-go v1.3.0 github.com/multiversx/mx-chain-core-go v1.4.1 github.com/multiversx/mx-chain-crypto-go v1.3.0 - github.com/multiversx/mx-chain-es-indexer-go v1.9.2 + github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 github.com/multiversx/mx-chain-storage-go v1.1.0 - github.com/multiversx/mx-chain-vm-common-go v1.6.5 + github.com/multiversx/mx-chain-vm-common-go v1.6.6 github.com/multiversx/mx-chain-vm-go v1.5.45 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 github.com/multiversx/mx-chain-vm-v1_3-go v1.3.70 diff --git a/go.sum b/go.sum index b1920975338..9caaf789bc5 100644 --- a/go.sum +++ b/go.sum @@ -405,16 +405,16 @@ github.com/multiversx/mx-chain-core-go v1.4.1 h1:ljs53jpdjtCohpaqm2n/dvTGrFlSgIp github.com/multiversx/mx-chain-core-go v1.4.1/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= -github.com/multiversx/mx-chain-es-indexer-go v1.9.2 h1:/K/cpTkwlFJ7zOD8VRhgc6ixi1t/3ua8CLl63LWHjvE= -github.com/multiversx/mx-chain-es-indexer-go v1.9.2/go.mod h1:t1rkD2vHXSI4EClig0h7+kRCSUCRrMF+emr4DHxFtfA= +github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= +github.com/multiversx/mx-chain-es-indexer-go v1.9.3/go.mod h1:dXRu2fmdiLFOcaRA34axQfoUcq8p9NUGqr4+9dN+p0Y= github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/WI0Qh112whgZNM= github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= -github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= -github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= +github.com/multiversx/mx-chain-vm-common-go v1.6.6 h1:BJSQndP8KSqcSIi47wQwQy3uBIn5rbT3213eJroVaog= +github.com/multiversx/mx-chain-vm-common-go v1.6.6/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= github.com/multiversx/mx-chain-vm-go v1.5.45/go.mod h1:Qc2Sckw+EfQwnapkzghFfhuUAOGv29oSZgvj8LJ+xWQ= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 h1:5gSR3IMw1mcp/v5oO+vZ5YOyWO8w7O2qKhCKNPwsWNE= diff --git a/integrationTests/testProcessorNodeWithTestWebServer.go b/integrationTests/testProcessorNodeWithTestWebServer.go index 792e43a5045..ec45aa10785 100644 --- a/integrationTests/testProcessorNodeWithTestWebServer.go +++ b/integrationTests/testProcessorNodeWithTestWebServer.go @@ -7,6 +7,7 @@ import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-vm-common-go/parsers" datafield "github.com/multiversx/mx-chain-vm-common-go/parsers/dataField" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" @@ -167,8 +168,9 @@ func createFacadeComponents(tpn *TestProcessorNode) nodeFacade.ApiResolver { log.LogIfError(err) argsDataFieldParser := &datafield.ArgsOperationDataFieldParser{ - AddressLength: TestAddressPubkeyConverter.Len(), - Marshalizer: TestMarshalizer, + AddressLength: TestAddressPubkeyConverter.Len(), + Marshalizer: TestMarshalizer, + RelayedTransactionsV1V2DisableEpoch: tpn.EnableEpochsHandler.GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), } dataFieldParser, err := datafield.NewOperationDataFieldParser(argsDataFieldParser) log.LogIfError(err) diff --git a/integrationTests/vm/testInitializer.go b/integrationTests/vm/testInitializer.go index 8e8944c6d6d..aadebd71402 100644 --- a/integrationTests/vm/testInitializer.go +++ b/integrationTests/vm/testInitializer.go @@ -995,8 +995,9 @@ func CreateTxProcessorWithOneSCExecutorWithVMs( }) dataFieldParser, err := datafield.NewOperationDataFieldParser(&datafield.ArgsOperationDataFieldParser{ - AddressLength: pubkeyConv.Len(), - Marshalizer: integrationtests.TestMarshalizer, + AddressLength: pubkeyConv.Len(), + Marshalizer: integrationtests.TestMarshalizer, + RelayedTransactionsV1V2DisableEpoch: enableEpochsHandler.GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), }) if err != nil { return nil, err diff --git a/node/external/transactionAPI/apiTransactionResults.go b/node/external/transactionAPI/apiTransactionResults.go index d4a89edfd15..a8a6af050f8 100644 --- a/node/external/transactionAPI/apiTransactionResults.go +++ b/node/external/transactionAPI/apiTransactionResults.go @@ -123,7 +123,7 @@ func (arp *apiTransactionResultsProcessor) getSmartContractResultsInTransactionB return nil, fmt.Errorf("%w: %v, hash = %s", errCannotLoadContractResults, err, hex.EncodeToString(scrHash)) } - scrAPI := arp.adaptSmartContractResult(scrHash, scr) + scrAPI := arp.adaptSmartContractResult(scrHash, scr, epoch) arp.loadLogsIntoContractResults(scrHash, epoch, scrAPI) @@ -171,7 +171,7 @@ func (arp *apiTransactionResultsProcessor) getScrFromStorage(hash []byte, epoch return scr, nil } -func (arp *apiTransactionResultsProcessor) adaptSmartContractResult(scrHash []byte, scr *smartContractResult.SmartContractResult) *transaction.ApiSmartContractResult { +func (arp *apiTransactionResultsProcessor) adaptSmartContractResult(scrHash []byte, scr *smartContractResult.SmartContractResult, epoch uint32) *transaction.ApiSmartContractResult { isRefund := arp.refundDetector.IsRefund(RefundDetectorInput{ Value: scr.Value.String(), Data: scr.Data, @@ -201,7 +201,7 @@ func (arp *apiTransactionResultsProcessor) adaptSmartContractResult(scrHash []by apiSCR.RelayerAddr, _ = arp.addressPubKeyConverter.Encode(scr.RelayerAddr) apiSCR.OriginalSender, _ = arp.addressPubKeyConverter.Encode(scr.OriginalSender) - res := arp.dataFieldParser.Parse(scr.Data, scr.GetSndAddr(), scr.GetRcvAddr(), arp.shardCoordinator.NumberOfShards()) + res := arp.dataFieldParser.Parse(scr.Data, scr.GetSndAddr(), scr.GetRcvAddr(), arp.shardCoordinator.NumberOfShards(), epoch) apiSCR.Operation = res.Operation apiSCR.Function = res.Function apiSCR.ESDTValues = res.ESDTValues diff --git a/node/external/transactionAPI/interface.go b/node/external/transactionAPI/interface.go index a32cac06184..cb64f15f9ab 100644 --- a/node/external/transactionAPI/interface.go +++ b/node/external/transactionAPI/interface.go @@ -28,5 +28,5 @@ type LogsFacade interface { // DataFieldParser defines what a data field parser should be able to do type DataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, epoch uint32) *datafield.ResponseParseData } diff --git a/node/external/transactionAPI/unmarshaller.go b/node/external/transactionAPI/unmarshaller.go index 42b2b21354c..ec5bc195c28 100644 --- a/node/external/transactionAPI/unmarshaller.go +++ b/node/external/transactionAPI/unmarshaller.go @@ -98,7 +98,7 @@ func (tu *txUnmarshaller) unmarshalTransaction( return nil, err } - res := tu.dataFieldParser.Parse(apiTx.Data, apiTx.Tx.GetSndAddr(), apiTx.Tx.GetRcvAddr(), tu.shardCoordinator.NumberOfShards()) + res := tu.dataFieldParser.Parse(apiTx.Data, apiTx.Tx.GetSndAddr(), apiTx.Tx.GetRcvAddr(), tu.shardCoordinator.NumberOfShards(), txEpoch) apiTx.Operation = res.Operation apiTx.Function = res.Function apiTx.ESDTValues = res.ESDTValues diff --git a/outport/process/transactionsfee/interface.go b/outport/process/transactionsfee/interface.go index 551ee59d1e2..b78f8e20741 100644 --- a/outport/process/transactionsfee/interface.go +++ b/outport/process/transactionsfee/interface.go @@ -23,5 +23,5 @@ type transactionGetter interface { } type dataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, epoch uint32) *datafield.ResponseParseData } diff --git a/outport/process/transactionsfee/transactionChecker.go b/outport/process/transactionsfee/transactionChecker.go index fd56d0c202b..830820c60f4 100644 --- a/outport/process/transactionsfee/transactionChecker.go +++ b/outport/process/transactionsfee/transactionChecker.go @@ -13,8 +13,8 @@ import ( vmcommon "github.com/multiversx/mx-chain-vm-common-go" ) -func (tep *transactionsFeeProcessor) isESDTOperationWithSCCall(tx data.TransactionHandler) bool { - res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards()) +func (tep *transactionsFeeProcessor) isESDTOperationWithSCCall(tx data.TransactionHandler, epoch uint32) bool { + res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards(), epoch) isESDTTransferOperation := res.Operation == core.BuiltInFunctionESDTTransfer || res.Operation == core.BuiltInFunctionESDTNFTTransfer || res.Operation == core.BuiltInFunctionMultiESDTNFTTransfer diff --git a/outport/process/transactionsfee/transactionsFeeProcessor.go b/outport/process/transactionsfee/transactionsFeeProcessor.go index 728d625cfa6..de9f5d61c68 100644 --- a/outport/process/transactionsfee/transactionsFeeProcessor.go +++ b/outport/process/transactionsfee/transactionsFeeProcessor.go @@ -51,8 +51,9 @@ func NewTransactionsFeeProcessor(arg ArgTransactionsFeeProcessor) (*transactions } parser, err := datafield.NewOperationDataFieldParser(&datafield.ArgsOperationDataFieldParser{ - AddressLength: arg.PubKeyConverter.Len(), - Marshalizer: arg.Marshaller, + AddressLength: arg.PubKeyConverter.Len(), + Marshalizer: arg.Marshaller, + RelayedTransactionsV1V2DisableEpoch: arg.EnableEpochsHandler.GetActivationEpoch(common.RelayedTransactionsV1V2DisableFlag), }) if err != nil { return nil, err @@ -131,7 +132,7 @@ func (tep *transactionsFeeProcessor) prepareNormalTxs(transactionsAndScrs *trans isRelayed := tep.isRelayedTxV1V2(txWithResult, epoch) isFeeFixActive := tep.enableEpochsHandler.IsFlagEnabledInEpoch(common.FixRelayedBaseCostFlag, epoch) isRelayedBeforeFix := isRelayed && !isFeeFixActive - if isRelayedBeforeFix || tep.isESDTOperationWithSCCall(txHandler) { + if isRelayedBeforeFix || tep.isESDTOperationWithSCCall(txHandler, epoch) { feeInfo.SetGasUsed(txWithResult.GetTxHandler().GetGasLimit()) feeInfo.SetFee(initialPaidFee) } @@ -259,7 +260,7 @@ func (tep *transactionsFeeProcessor) prepareTxWithResultsBasedOnLogs( return } - res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards()) + res := tep.dataFieldParser.Parse(tx.GetData(), tx.GetSndAddr(), tx.GetRcvAddr(), tep.shardCoordinator.NumberOfShards(), epoch) if check.IfNilReflect(txWithResults.log) || (res.Function == "" && res.Operation == datafield.OperationTransfer) { return } diff --git a/process/transactionEvaluator/interface.go b/process/transactionEvaluator/interface.go index 0b6d2620d72..979e7098c61 100644 --- a/process/transactionEvaluator/interface.go +++ b/process/transactionEvaluator/interface.go @@ -15,5 +15,5 @@ type TransactionProcessor interface { // DataFieldParser defines what a data field parser should be able to do type DataFieldParser interface { - Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData + Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, epoch uint32) *datafield.ResponseParseData } diff --git a/process/transactionEvaluator/transactionSimulator.go b/process/transactionEvaluator/transactionSimulator.go index d01dc4ef85d..1fea49a2ac2 100644 --- a/process/transactionEvaluator/transactionSimulator.go +++ b/process/transactionEvaluator/transactionSimulator.go @@ -270,7 +270,9 @@ func (ts *transactionSimulator) adaptSmartContractResult(scr *smartContractResul ReturnMessage: string(scr.ReturnMessage), GasLimit: scr.GasLimit, }) - res := ts.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, ts.shardCoordinator.NumberOfShards()) + + currentEpoch := ts.blockChainHook.CurrentEpoch() + res := ts.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, ts.shardCoordinator.NumberOfShards(), currentEpoch) receiversEncoded, err := ts.addressPubKeyConverter.EncodeSlice(res.Receivers) if err != nil { diff --git a/testscommon/dataFieldParserStub.go b/testscommon/dataFieldParserStub.go index fcbe84497c7..f40a9117c59 100644 --- a/testscommon/dataFieldParserStub.go +++ b/testscommon/dataFieldParserStub.go @@ -8,7 +8,7 @@ type DataFieldParserStub struct { } // Parse - -func (df *DataFieldParserStub) Parse(dataField []byte, sender, receiver []byte, numOfShards uint32) *datafield.ResponseParseData { +func (df *DataFieldParserStub) Parse(dataField []byte, sender, receiver []byte, numOfShards uint32, _ uint32) *datafield.ResponseParseData { if df.ParseCalled != nil { return df.ParseCalled(dataField, sender, receiver, numOfShards) } From a73bf237cfabcde9430689ee69017a77fc76b1ca Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 29 Apr 2026 18:16:29 +0300 Subject: [PATCH 023/116] add resolver exception recover --- .../resolvers/equivalentProofsResolver.go | 13 ++++- dataRetriever/resolvers/headerResolver.go | 15 ++++-- dataRetriever/resolvers/miniblockResolver.go | 12 ++++- .../resolvers/peerAuthenticationResolver.go | 12 ++++- .../resolvers/transactionResolver.go | 12 ++++- dataRetriever/resolvers/trieNodeResolver.go | 13 ++++- .../resolvers/trieNodeResolver_test.go | 51 +++++++++++++++++++ .../resolvers/validatorInfoResolver.go | 12 ++++- 8 files changed, 125 insertions(+), 15 deletions(-) diff --git a/dataRetriever/resolvers/equivalentProofsResolver.go b/dataRetriever/resolvers/equivalentProofsResolver.go index c36c3e9ac92..b2e51da87b4 100644 --- a/dataRetriever/resolvers/equivalentProofsResolver.go +++ b/dataRetriever/resolvers/equivalentProofsResolver.go @@ -2,6 +2,8 @@ package resolvers import ( "fmt" + "runtime/debug" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" @@ -90,8 +92,15 @@ func checkArgEquivalentProofsResolver(args ArgEquivalentProofsResolver) error { // ProcessReceivedMessage represents the callback func from the p2p.Messenger that is called each time a new message is received // (for the topic this validator was registered to, usually a request topic) -func (res *equivalentProofsResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := res.canProcessMessage(message, fromConnectedPeer) +func (res *equivalentProofsResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in equivalentProofsResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = res.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/headerResolver.go b/dataRetriever/resolvers/headerResolver.go index dbd8626bf3a..99eb50dbde0 100644 --- a/dataRetriever/resolvers/headerResolver.go +++ b/dataRetriever/resolvers/headerResolver.go @@ -1,12 +1,14 @@ package resolvers import ( + "fmt" + "runtime/debug" "sync" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/typeConverters" - "github.com/multiversx/mx-chain-logger-go" + logger "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/resolvers/epochproviders/disabled" @@ -110,8 +112,15 @@ func (hdrRes *HeaderResolver) SetEpochHandler(epochHandler dataRetriever.EpochHa // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (hdrRes *HeaderResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := hdrRes.canProcessMessage(message, fromConnectedPeer) +func (hdrRes *HeaderResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in HeaderResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = hdrRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/miniblockResolver.go b/dataRetriever/resolvers/miniblockResolver.go index 3fb74105af5..0909a2efb47 100644 --- a/dataRetriever/resolvers/miniblockResolver.go +++ b/dataRetriever/resolvers/miniblockResolver.go @@ -2,6 +2,7 @@ package resolvers import ( "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -78,8 +79,15 @@ func checkArgMiniblockResolver(arg ArgMiniblockResolver) error { // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (mbRes *miniblockResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := mbRes.canProcessMessage(message, fromConnectedPeer) +func (mbRes *miniblockResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in miniblockResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = mbRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/peerAuthenticationResolver.go b/dataRetriever/resolvers/peerAuthenticationResolver.go index 49f29ff0246..14091ca7698 100644 --- a/dataRetriever/resolvers/peerAuthenticationResolver.go +++ b/dataRetriever/resolvers/peerAuthenticationResolver.go @@ -2,6 +2,7 @@ package resolvers import ( "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -76,8 +77,15 @@ func checkArgPeerAuthenticationResolver(arg ArgPeerAuthenticationResolver) error // ProcessReceivedMessage represents the callback func from the p2p.Messenger that is called each time a new message is received // (for the topic this validator was registered to, usually a request topic) -func (res *peerAuthenticationResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := res.canProcessMessage(message, fromConnectedPeer) +func (res *peerAuthenticationResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in peerAuthenticationResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = res.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/transactionResolver.go b/dataRetriever/resolvers/transactionResolver.go index 8495c970a70..4d79277f260 100644 --- a/dataRetriever/resolvers/transactionResolver.go +++ b/dataRetriever/resolvers/transactionResolver.go @@ -2,6 +2,7 @@ package resolvers import ( "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -83,8 +84,15 @@ func checkArgTxResolver(arg ArgTxResolver) error { // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (txRes *TxResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := txRes.canProcessMessage(message, fromConnectedPeer) +func (txRes *TxResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in TxResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = txRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/trieNodeResolver.go b/dataRetriever/resolvers/trieNodeResolver.go index 78ed24d0159..721105b64d7 100644 --- a/dataRetriever/resolvers/trieNodeResolver.go +++ b/dataRetriever/resolvers/trieNodeResolver.go @@ -1,6 +1,8 @@ package resolvers import ( + "fmt" + "runtime/debug" "sync" "github.com/multiversx/mx-chain-core-go/core" @@ -63,8 +65,15 @@ func checkArgTrieNodeResolver(arg ArgTrieNodeResolver) error { // ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received // (for the topic this validator was registered to, usually a request topic) -func (tnRes *TrieNodeResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := tnRes.canProcessMessage(message, fromConnectedPeer) +func (tnRes *TrieNodeResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in TrieNodeResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = tnRes.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } diff --git a/dataRetriever/resolvers/trieNodeResolver_test.go b/dataRetriever/resolvers/trieNodeResolver_test.go index b988b2f2959..6347f02875a 100644 --- a/dataRetriever/resolvers/trieNodeResolver_test.go +++ b/dataRetriever/resolvers/trieNodeResolver_test.go @@ -587,6 +587,57 @@ func TestTrieNodeResolver_ProcessReceivedMessageLargeTrieNodeShouldSendFirstChun testTrieNodeResolverProcessReceivedMessageLargeTrieNode(t, randBuff, 0, 4, 0, core.MaxBufferSizeToSendTrieNodes) } +func TestTrieNodeResolver_ProcessReceivedMessageLargeTrieNodeMaxChunkIndex(t *testing.T) { + t.Parallel() + + largeBuffer := make([]byte, 393216) // 256k + 128k + chunkIndex := uint32(2) + + nodes := [][]byte{largeBuffer} + hashes := [][]byte{[]byte("hash1")} + + sendWasCalled := false + arg := createMockArgTrieNodeResolver() + arg.SenderResolver = &mock.TopicResolverSenderStub{ + SendCalled: func(buff []byte, peer core.PeerID, source p2p.MessageHandler) error { + sendWasCalled = true + return nil + }, + } + arg.TrieDataGetter = &trieMock.TrieStub{ + GetSerializedNodeCalled: func(hash []byte) ([]byte, error) { + for i := 0; i < len(hashes); i++ { + if bytes.Equal(hash, hashes[i]) { + return nodes[i], nil + } + } + + return nil, fmt.Errorf("not found") + }, + GetSerializedNodesCalled: func(i []byte, u uint64) ([][]byte, uint64, error) { + return make([][]byte, 0), 0, nil + }, + } + tnRes, _ := resolvers.NewTrieNodeResolver(arg) + + data, _ := arg.Marshaller.Marshal( + &dataRetriever.RequestData{ + Type: dataRetriever.HashType, + Value: []byte("hash1"), + ChunkIndex: chunkIndex, + }, + ) + msg := &p2pmocks.P2PMessageMock{DataField: data} + + msgID, err := tnRes.ProcessReceivedMessage(msg, fromConnectedPeer, &p2pmocks.MessengerStub{}) + assert.Nil(t, err) + require.False(t, sendWasCalled) + assert.Len(t, msgID, 0) + + assert.True(t, arg.Throttler.(*mock.ThrottlerStub).StartWasCalled()) + assert.True(t, arg.Throttler.(*mock.ThrottlerStub).EndWasCalled()) +} + func TestTrieNodeResolver_ProcessReceivedMessageLargeTrieNodeShouldSendRequiredChunk(t *testing.T) { t.Parallel() diff --git a/dataRetriever/resolvers/validatorInfoResolver.go b/dataRetriever/resolvers/validatorInfoResolver.go index 65255b8ad8f..f3cb291abef 100644 --- a/dataRetriever/resolvers/validatorInfoResolver.go +++ b/dataRetriever/resolvers/validatorInfoResolver.go @@ -3,6 +3,7 @@ package resolvers import ( "encoding/hex" "fmt" + "runtime/debug" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -90,8 +91,15 @@ func checkArgs(args ArgValidatorInfoResolver) error { // ProcessReceivedMessage represents the callback func from the p2p.Messenger that is called each time a new message is received // (for the topic this validator was registered to, usually a request topic) -func (res *validatorInfoResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) ([]byte, error) { - err := res.canProcessMessage(message, fromConnectedPeer) +func (res *validatorInfoResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) (msg []byte, err error) { + defer func() { + if r := recover(); r != nil { + logTrieNodes.Error("panic recovered", "peer", fromConnectedPeer, "panic", r, "stack", string(debug.Stack())) + err = fmt.Errorf("panic in validatorInfoResolver.ProcessReceivedMessage: %v", r) + } + }() + + err = res.canProcessMessage(message, fromConnectedPeer) if err != nil { return nil, err } From 46deaeff669bbc0f291ea70b7c363546113490b0 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 30 Apr 2026 16:01:17 +0300 Subject: [PATCH 024/116] fix #137 --- consensus/spos/worker.go | 10 +++++----- consensus/spos/worker_test.go | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 86cd3b39dfa..e8dbec25648 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -506,6 +506,11 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP wrk.consensusState.ResetRoundsWithoutReceivedMessages(cnsMsg.GetPubKey(), message.Peer()) + err = wrk.checkValidityAndProcessFinalInfo(cnsMsg, message) + if err != nil { + return nil, err + } + if wrk.nodeRedundancyHandler.IsRedundancyNode() { wrk.nodeRedundancyHandler.ResetInactivityIfNeeded( wrk.consensusState.SelfPubKey(), @@ -514,11 +519,6 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP ) } - err = wrk.checkValidityAndProcessFinalInfo(cnsMsg, message) - if err != nil { - return nil, err - } - wrk.networkShardingCollector.UpdatePeerIDInfo(message.Peer(), cnsMsg.PubKey, wrk.shardCoordinator.SelfId()) msgType := consensus.MessageType(cnsMsg.MsgType) diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index a144b88dcff..18bbc2c7ac4 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -607,10 +607,30 @@ func TestWorker_ProcessReceivedMessageRedundancyNodeShouldResetInactivityIfNeede }, } wrk.SetNodeRedundancyHandler(nodeRedundancyMock) - buff, _ := wrk.Marshalizer().Marshal(&consensus.Message{}) + hdr := &block.Header{ChainID: chainID} + hdrHash, _ := core.CalculateHash(mock.MarshalizerMock{}, &hashingMocks.HasherMock{}, hdr) + hdrStr, _ := mock.MarshalizerMock{}.Marshal(hdr) + cnsMsg := consensus.NewConsensusMessage( + hdrHash, + nil, + nil, + hdrStr, + []byte(wrk.ConsensusState().ConsensusGroup()[0]), + signature, + int(bls.MtBlockHeader), + 0, + chainID, + nil, + nil, + nil, + currentPid, + nil, + ) + buff, _ := wrk.Marshalizer().Marshal(cnsMsg) _, _ = wrk.ProcessReceivedMessage( &p2pmocks.P2PMessageMock{ DataField: buff, + PeerField: currentPid, SignatureField: []byte("signature"), }, fromConnectedPeerId, From bab319951460a76a4af3bf4fb05509dc54be0164 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 4 May 2026 15:05:41 +0300 Subject: [PATCH 025/116] UpdatePeerIDPublicKeyPair now gets the timestamp from payload --- consensus/spos/worker.go | 4 +-- .../disabled/disabledPeerShardMapper.go | 2 +- go.mod | 2 +- go.sum | 4 +-- integrationTests/interface.go | 2 +- .../mock/networkShardingCollectorMock.go | 2 +- integrationTests/mock/peerShardMapperStub.go | 6 ++--- .../interceptedPeerAuthentication.go | 22 +++++++++------- .../interceptedPeerAuthentication_test.go | 26 +++++++++++++++++++ .../peerAuthenticationInterceptorProcessor.go | 6 ++--- ...AuthenticationInterceptorProcessor_test.go | 4 +-- process/interface.go | 2 +- process/mock/peerShardMapperStub.go | 6 ++--- sharding/networksharding/peerShardMapper.go | 2 +- .../networksharding/peerShardMapper_test.go | 2 +- .../p2pmocks/networkShardingCollectorStub.go | 6 ++--- 16 files changed, 64 insertions(+), 34 deletions(-) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index e8dbec25648..5599b535f0f 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -504,13 +504,13 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP return nil, err } - wrk.consensusState.ResetRoundsWithoutReceivedMessages(cnsMsg.GetPubKey(), message.Peer()) - err = wrk.checkValidityAndProcessFinalInfo(cnsMsg, message) if err != nil { return nil, err } + wrk.consensusState.ResetRoundsWithoutReceivedMessages(cnsMsg.GetPubKey(), message.Peer()) + if wrk.nodeRedundancyHandler.IsRedundancyNode() { wrk.nodeRedundancyHandler.ResetInactivityIfNeeded( wrk.consensusState.SelfPubKey(), diff --git a/epochStart/bootstrap/disabled/disabledPeerShardMapper.go b/epochStart/bootstrap/disabled/disabledPeerShardMapper.go index c4695c00c09..a0b68087db3 100644 --- a/epochStart/bootstrap/disabled/disabledPeerShardMapper.go +++ b/epochStart/bootstrap/disabled/disabledPeerShardMapper.go @@ -17,7 +17,7 @@ func (p *peerShardMapper) GetLastKnownPeerID(_ []byte) (core.PeerID, bool) { } // UpdatePeerIDPublicKeyPair does nothing -func (p *peerShardMapper) UpdatePeerIDPublicKeyPair(_ core.PeerID, _ []byte) { +func (p *peerShardMapper) UpdatePeerIDPublicKeyPair(_ core.PeerID, _ []byte, _ int64) { } // PutPeerIdShardId does nothing diff --git a/go.mod b/go.mod index 85e957d1889..d5fcb8821d9 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.3.0 - github.com/multiversx/mx-chain-core-go v1.4.1 + github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a github.com/multiversx/mx-chain-crypto-go v1.3.0 github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 diff --git a/go.sum b/go.sum index 9caaf789bc5..252b93e81e5 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.1 h1:ljs53jpdjtCohpaqm2n/dvTGrFlSgIpoZYH8RVt5cWo= -github.com/multiversx/mx-chain-core-go v1.4.1/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a h1:UWVheMivOd2M7XEhZjlUGpI/V8skd8MfYv0JngpH98E= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= diff --git a/integrationTests/interface.go b/integrationTests/interface.go index 23504565a25..6c7c69103d1 100644 --- a/integrationTests/interface.go +++ b/integrationTests/interface.go @@ -49,7 +49,7 @@ type NodesCoordinatorFactory interface { // NetworkShardingUpdater defines the updating methods used by the network sharding component type NetworkShardingUpdater interface { GetPeerInfo(pid core.PeerID) core.P2PPeerInfo - UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdShardId(pid core.PeerID, shardID uint32) UpdatePeerIDInfo(pid core.PeerID, pk []byte, shardID uint32) PutPeerIdSubType(pid core.PeerID, peerSubType core.P2PPeerSubType) diff --git a/integrationTests/mock/networkShardingCollectorMock.go b/integrationTests/mock/networkShardingCollectorMock.go index cfd163e88ea..d3c8a17901a 100644 --- a/integrationTests/mock/networkShardingCollectorMock.go +++ b/integrationTests/mock/networkShardingCollectorMock.go @@ -33,7 +33,7 @@ func NewNetworkShardingCollectorMock() *networkShardingCollectorMock { } // UpdatePeerIDPublicKeyPair - -func (nscm *networkShardingCollectorMock) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (nscm *networkShardingCollectorMock) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, _ int64) { nscm.mutMaps.Lock() nscm.peerIdPkMap[pid] = pk nscm.pkPeerIdMap[string(pk)] = pid diff --git a/integrationTests/mock/peerShardMapperStub.go b/integrationTests/mock/peerShardMapperStub.go index b32a1045c7b..4f49be5a4f9 100644 --- a/integrationTests/mock/peerShardMapperStub.go +++ b/integrationTests/mock/peerShardMapperStub.go @@ -5,7 +5,7 @@ import "github.com/multiversx/mx-chain-core-go/core" // PeerShardMapperStub - type PeerShardMapperStub struct { GetLastKnownPeerIDCalled func(pk []byte) (core.PeerID, bool) - UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdShardIdCalled func(pid core.PeerID, shardID uint32) PutPeerIdSubTypeCalled func(pid core.PeerID, peerSubType core.P2PPeerSubType) UpdatePeerIDInfoCalled func(pid core.PeerID, pk []byte, shardID uint32) @@ -19,9 +19,9 @@ func (psms *PeerShardMapperStub) UpdatePeerIDInfo(pid core.PeerID, pk []byte, sh } // UpdatePeerIDPublicKeyPair - -func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { if psms.UpdatePeerIDPublicKeyPairCalled != nil { - psms.UpdatePeerIDPublicKeyPairCalled(pid, pk) + psms.UpdatePeerIDPublicKeyPairCalled(pid, pk, timestamp) } } diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 9a205c56dda..2af87dfdd4c 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -139,22 +139,26 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { if err != nil { return err } - } - // Early exit if mapping already exists - existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) - if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { - return process.ErrPeerAlreadyAuthenticated + // Early exit if mapping already exists + existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) + if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { + return process.ErrPeerAlreadyAuthenticated + } + + if existingInfo.AuthenticationTimestamp > ipa.payload.Timestamp { + return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthenticationTimestamp) + } } - // Verify payload signature - err = ipa.signaturesHandler.Verify(ipa.peerAuthentication.Payload, ipa.peerId, ipa.peerAuthentication.PayloadSignature) + // Verify payload + err = ipa.payloadValidator.ValidateTimestamp(ipa.payload.Timestamp) if err != nil { return err } - // Verify payload - err = ipa.payloadValidator.ValidateTimestamp(ipa.payload.Timestamp) + // Verify payload signature + err = ipa.signaturesHandler.Verify(ipa.peerAuthentication.Payload, ipa.peerId, ipa.peerAuthentication.PayloadSignature) if err != nil { return err } diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index a48c3d4c7fd..4569371f1ed 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -292,6 +292,32 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) }) + t.Run("peer already authenticated with newer timestamp should return error", func(t *testing.T) { + t.Parallel() + + providedPA := createDefaultInterceptedPeerAuthentication() + + arg := createMockInterceptedPeerAuthenticationArg(providedPA) + + authTimestamp := time.Now().Add(time.Minute).Unix() + arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ + VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { + require.Fail(t, "should have not been called") + return expectedErr + }, + } + arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + return core.P2PPeerInfo{ + AuthenticationTimestamp: authTimestamp, + } + }, + } + + ipa, _ := NewInterceptedPeerAuthentication(arg) + err := ipa.CheckValidity() + assert.ErrorIs(t, err, process.ErrPeerAlreadyAuthenticated) + }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go index 5864dcfcbf8..1c8950e2a8b 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor.go @@ -82,10 +82,10 @@ func (paip *peerAuthenticationInterceptorProcessor) Save(data process.Intercepte return err } - return paip.updatePeerInfo(interceptedPeerAuthenticationData.Message(), interceptedPeerAuthenticationData.SizeInBytes()) + return paip.updatePeerInfo(interceptedPeerAuthenticationData.Message(), interceptedPeerAuthenticationData.SizeInBytes(), payload.Timestamp) } -func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message interface{}, messageSize int) error { +func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message interface{}, messageSize int, payloadTimestamp int64) error { peerAuthenticationData, ok := message.(*heartbeat.PeerAuthentication) if !ok { return process.ErrWrongTypeAssertion @@ -93,7 +93,7 @@ func (paip *peerAuthenticationInterceptorProcessor) updatePeerInfo(message inter pidBytes := peerAuthenticationData.GetPid() paip.peerAuthenticationCacher.Put(peerAuthenticationData.Pubkey, message, messageSize) - paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey()) + paip.peerShardMapper.UpdatePeerIDPublicKeyPair(core.PeerID(pidBytes), peerAuthenticationData.GetPubkey(), payloadTimestamp) log.Trace("PeerAuthentication message saved") diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 09016fbc0af..d941cb79df1 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -136,7 +136,7 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { wasCalled := false args := createPeerAuthenticationInterceptorProcessArg() args.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ - UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { + UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte, timestamp int64) { wasCalled = true }, } @@ -205,7 +205,7 @@ func TestPeerAuthenticationInterceptorProcessor_Save(t *testing.T) { } wasUpdatePeerIDPublicKeyPairCalled := false arg.PeerShardMapper = &p2pmocks.NetworkShardingCollectorStub{ - UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte) { + UpdatePeerIDPublicKeyPairCalled: func(pid core.PeerID, pk []byte, timestamp int64) { wasUpdatePeerIDPublicKeyPairCalled = true assert.Equal(t, providedIPAMessage.Pid, pid.Bytes()) assert.Equal(t, providedIPAMessage.Pubkey, pk) diff --git a/process/interface.go b/process/interface.go index 99bafaa1354..eb0feedbf34 100644 --- a/process/interface.go +++ b/process/interface.go @@ -789,7 +789,7 @@ type PeerBlackListCacher interface { // PeerShardMapper can return the public key of a provided peer ID type PeerShardMapper interface { - UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdShardId(pid core.PeerID, shardID uint32) PutPeerIdSubType(pid core.PeerID, peerSubType core.P2PPeerSubType) GetPeerInfo(pid core.PeerID) core.P2PPeerInfo diff --git a/process/mock/peerShardMapperStub.go b/process/mock/peerShardMapperStub.go index 8c73a582904..364c19dd0ab 100644 --- a/process/mock/peerShardMapperStub.go +++ b/process/mock/peerShardMapperStub.go @@ -9,7 +9,7 @@ type PeerShardMapperStub struct { UpdatePeerIdPublicKeyCalled func(pid core.PeerID, pk []byte) UpdatePublicKeyShardIdCalled func(pk []byte, shardId uint32) PutPeerIdShardIdCalled func(pid core.PeerID, shardId uint32) - UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte, timestamp int64) PutPeerIdSubTypeCalled func(pid core.PeerID, peerSubType core.P2PPeerSubType) } @@ -32,9 +32,9 @@ func (psms *PeerShardMapperStub) GetPeerInfo(pid core.PeerID) core.P2PPeerInfo { } // UpdatePeerIDPublicKeyPair - -func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (psms *PeerShardMapperStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { if psms.UpdatePeerIDPublicKeyPairCalled != nil { - psms.UpdatePeerIDPublicKeyPairCalled(pid, pk) + psms.UpdatePeerIDPublicKeyPairCalled(pid, pk, timestamp) } } diff --git a/sharding/networksharding/peerShardMapper.go b/sharding/networksharding/peerShardMapper.go index f66bbf52742..71def8d4598 100644 --- a/sharding/networksharding/peerShardMapper.go +++ b/sharding/networksharding/peerShardMapper.go @@ -231,7 +231,7 @@ func (psm *PeerShardMapper) getPeerInfoSearchingPidInFallbackCache(pid core.Peer // UpdatePeerIDPublicKeyPair updates the public key - peer ID pair in the corresponding maps // It also uses the intermediate pkPeerId cache that will prevent having thousands of peer ID's with // the same MultiversX PK that will make the node prone to an eclipse attack -func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, _ int64) { isNew := psm.updatePeerIDPublicKey(pid, pk) if isNew { peerLog.Trace("new peer mapping", "pid", pid.Pretty(), "pk", pk) diff --git a/sharding/networksharding/peerShardMapper_test.go b/sharding/networksharding/peerShardMapper_test.go index 6b03abe6805..17cdbd47464 100644 --- a/sharding/networksharding/peerShardMapper_test.go +++ b/sharding/networksharding/peerShardMapper_test.go @@ -231,7 +231,7 @@ func TestPeerShardMapper_UpdatePeerIDPublicKeyPairShouldWork(t *testing.T) { pid := core.PeerID("dummy peer ID") pk := []byte("dummy pk") - psm.UpdatePeerIDPublicKeyPair(pid, pk) + psm.UpdatePeerIDPublicKeyPair(pid, pk, 0) pkRecovered := psm.GetPkFromPidPk(pid) assert.Equal(t, pk, pkRecovered) diff --git a/testscommon/p2pmocks/networkShardingCollectorStub.go b/testscommon/p2pmocks/networkShardingCollectorStub.go index b7b1d3fb21b..bee8f438d46 100644 --- a/testscommon/p2pmocks/networkShardingCollectorStub.go +++ b/testscommon/p2pmocks/networkShardingCollectorStub.go @@ -6,7 +6,7 @@ import ( // NetworkShardingCollectorStub - type NetworkShardingCollectorStub struct { - UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte) + UpdatePeerIDPublicKeyPairCalled func(pid core.PeerID, pk []byte, timestamp int64) UpdatePeerIDInfoCalled func(pid core.PeerID, pk []byte, shardID uint32) PutPeerIdShardIdCalled func(pid core.PeerID, shardId uint32) PutPeerIdSubTypeCalled func(pid core.PeerID, peerSubType core.P2PPeerSubType) @@ -15,9 +15,9 @@ type NetworkShardingCollectorStub struct { } // UpdatePeerIDPublicKeyPair - -func (nscs *NetworkShardingCollectorStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte) { +func (nscs *NetworkShardingCollectorStub) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { if nscs.UpdatePeerIDPublicKeyPairCalled != nil { - nscs.UpdatePeerIDPublicKeyPairCalled(pid, pk) + nscs.UpdatePeerIDPublicKeyPairCalled(pid, pk, timestamp) } } From c1d246cc7209d767ae28515fefac0bcf6ebab83e Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Tue, 5 May 2026 11:35:31 +0300 Subject: [PATCH 026/116] finish implementation on psm --- go.mod | 2 +- go.sum | 4 +-- .../interceptedPeerAuthentication.go | 4 +-- .../interceptedPeerAuthentication_test.go | 2 +- sharding/networksharding/peerShardMapper.go | 33 ++++++++++++++++--- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index d5fcb8821d9..70cb5650b0a 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.3.0 - github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a + github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 github.com/multiversx/mx-chain-crypto-go v1.3.0 github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 diff --git a/go.sum b/go.sum index 252b93e81e5..f5a56769576 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a h1:UWVheMivOd2M7XEhZjlUGpI/V8skd8MfYv0JngpH98E= -github.com/multiversx/mx-chain-core-go v1.4.2-0.20260504093908-b862c80a725a/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 h1:hpnYOT5cDJip7B6GvFRSOcUdwdhBbuFsNjLvOxzYOn8= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 2af87dfdd4c..f1b43d294a9 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -146,8 +146,8 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { return process.ErrPeerAlreadyAuthenticated } - if existingInfo.AuthenticationTimestamp > ipa.payload.Timestamp { - return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthenticationTimestamp) + if existingInfo.AuthTimestamp > ipa.payload.Timestamp { + return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthTimestamp) } } diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 4569371f1ed..84ae30f5c48 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -309,7 +309,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { return core.P2PPeerInfo{ - AuthenticationTimestamp: authTimestamp, + AuthTimestamp: authTimestamp, } }, } diff --git a/sharding/networksharding/peerShardMapper.go b/sharding/networksharding/peerShardMapper.go index 71def8d4598..4e2fd1bfd5f 100644 --- a/sharding/networksharding/peerShardMapper.go +++ b/sharding/networksharding/peerShardMapper.go @@ -18,6 +18,7 @@ import ( const maxNumPidsPerPk = 3 const uint32Size = 4 +const int64Size = 8 const defaultShardId = uint32(0) const indexNotFound = -1 @@ -38,6 +39,7 @@ var _ p2p.PeerShardResolver = (*PeerShardMapper)(nil) type PeerShardMapper struct { peerIdPkCache storage.Cacher pkPeerIdCache storage.Cacher + pkTimestampCache storage.Cacher fallbackPkShardCache storage.Cacher fallbackPidShardCache storage.Cacher peerIdSubTypeCache storage.Cacher @@ -85,9 +87,15 @@ func NewPeerShardMapper(arg ArgPeerShardMapper) (*PeerShardMapper, error) { return nil, err } + pkTimestamp, err := cache.NewLRUCache(arg.PeerIdPkCache.MaxSize()) + if err != nil { + return nil, err + } + return &PeerShardMapper{ peerIdPkCache: arg.PeerIdPkCache, pkPeerIdCache: pkPeerId, + pkTimestampCache: pkTimestamp, fallbackPkShardCache: arg.FallbackPkShardCache, fallbackPidShardCache: arg.FallbackPidShardCache, peerIdSubTypeCache: peerIdSubTypeCache, @@ -161,12 +169,27 @@ func (psm *PeerShardMapper) getPeerInfoWithNodesCoordinator(pid core.PeerID) (*c } return &core.P2PPeerInfo{ - PeerType: core.ValidatorPeer, - ShardID: shardId, - PkBytes: pkBuff, + PeerType: core.ValidatorPeer, + ShardID: shardId, + PkBytes: pkBuff, + AuthTimestamp: psm.getTimestampForPk(pkBuff), }, true } +func (psm *PeerShardMapper) getTimestampForPk(pkBytes []byte) int64 { + timestamp, ok := psm.pkTimestampCache.Get(pkBytes) + if !ok { + return 0 + } + + timestampInt, ok := timestamp.(int64) + if !ok { + return 0 + } + + return timestampInt +} + func (psm *PeerShardMapper) getShardIDSearchingPkInFallbackCache(pkBuff []byte) (shardId uint32, ok bool) { if len(pkBuff) == 0 { return defaultShardId, false @@ -231,11 +254,13 @@ func (psm *PeerShardMapper) getPeerInfoSearchingPidInFallbackCache(pid core.Peer // UpdatePeerIDPublicKeyPair updates the public key - peer ID pair in the corresponding maps // It also uses the intermediate pkPeerId cache that will prevent having thousands of peer ID's with // the same MultiversX PK that will make the node prone to an eclipse attack -func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, _ int64) { +func (psm *PeerShardMapper) UpdatePeerIDPublicKeyPair(pid core.PeerID, pk []byte, timestamp int64) { isNew := psm.updatePeerIDPublicKey(pid, pk) if isNew { peerLog.Trace("new peer mapping", "pid", pid.Pretty(), "pk", pk) } + + psm.pkTimestampCache.Put(pk, timestamp, int64Size) } // UpdatePeerIDInfo updates the public keys and the shard ID for the peer ID in the corresponding maps From 2e1dcfcdac32265806010cfc4d425942f8911445 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 6 May 2026 15:45:41 +0300 Subject: [PATCH 027/116] add more checks --- process/common.go | 94 ++++++++++++++++------ process/common_test.go | 171 ++++++++++++++++++++++++++++++----------- 2 files changed, 197 insertions(+), 68 deletions(-) diff --git a/process/common.go b/process/common.go index b1342c2148b..7764ac68103 100644 --- a/process/common.go +++ b/process/common.go @@ -1174,30 +1174,9 @@ func CheckMiniBlock( miniBlock.ReceiverShardID) } - // type checks - if miniBlock.GetType() == block.PeerBlock && - (miniBlock.GetSenderShardID() != core.MetachainShardId || miniBlock.GetReceiverShardID() != core.AllShardId) { - return fmt.Errorf("%w - peer blocks: block type: %s, sender shard id: %d, receiver shard id: %d", - ErrInvalidShardId, - miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) - } - - if miniBlock.GetType() != block.PeerBlock && miniBlock.GetReceiverShardID() == core.AllShardId { - return fmt.Errorf("%w - invalid all shard ids: block type: %s, sender shard id: %d, receiver shard id: %d", - ErrInvalidShardId, - miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) - } - - if miniBlock.GetType() == block.RewardsBlock && miniBlock.GetSenderShardID() != core.MetachainShardId { - return fmt.Errorf("%w - invalid rewards block: block type: %s, sender shard id: %d, receiver shard id: %d", - ErrInvalidShardId, - miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) + err := checkMiniBlockByType(miniBlock, shardCoordinator) + if err != nil { + return err } for _, txHash := range miniBlock.TxHashes { @@ -1212,3 +1191,70 @@ func CheckMiniBlock( return nil } + +func checkMiniBlockByType( + miniBlock *block.MiniBlock, + shardCoordinator sharding.Coordinator, +) error { + selfId := shardCoordinator.SelfId() + sender := miniBlock.GetSenderShardID() + receiver := miniBlock.GetReceiverShardID() + mbType := miniBlock.GetType() + + switch mbType { + case block.TxBlock: + if sender == core.MetachainShardId || receiver == core.AllShardId { + return fmt.Errorf("%w - TxBlock must be from shard to specific shard id: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + mbType, + sender, + receiver, + ) + } + + case block.SmartContractResultBlock: + if receiver == core.AllShardId { + return fmt.Errorf("%w - SCResultBlock cannot target AllShardId: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + mbType, + sender, + receiver, + ) + } + + case block.InvalidBlock, block.ReceiptBlock: + if sender != selfId || receiver != selfId { + return fmt.Errorf("%w - must be intra-shard: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + mbType, + sender, + receiver, + ) + } + + case block.PeerBlock: + if sender != core.MetachainShardId || receiver != core.AllShardId { + return fmt.Errorf("%w - PeerBlock must be from metachain to all shards: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + mbType, + sender, + receiver, + ) + } + + case block.RewardsBlock: + if sender != core.MetachainShardId || receiver == core.AllShardId { + return fmt.Errorf("%w - RewardsBlock must be from metachain to specific shard: block type: %s, sender shard id: %d, receiver shard id: %d", + ErrInvalidShardId, + mbType, + sender, + receiver, + ) + } + + default: + return fmt.Errorf("%w - unknown miniblock type %d", ErrInvalidShardId, int32(mbType)) + } + + return nil +} diff --git a/process/common_test.go b/process/common_test.go index c9dd3477938..f34198376db 100644 --- a/process/common_test.go +++ b/process/common_test.go @@ -2368,19 +2368,20 @@ func TestShardedCacheSearchMethod_ToString(t *testing.T) { func TestCheckMiniBlock(t *testing.T) { t.Parallel() + selfShardID := uint32(1) + + shardCoordinator := &mock.ShardCoordinatorStub{ + SelfIdCalled: func() uint32 { + return selfShardID + }, + NumberOfShardsCalled: func() uint32 { + return 3 + }, + } + t.Run("not related to self shard, should fail", func(t *testing.T) { t.Parallel() - selfShardID := uint32(1) - shardCoordinator := &mock.ShardCoordinatorStub{ - SelfIdCalled: func() uint32 { - return selfShardID - }, - NumberOfShardsCalled: func() uint32 { - return 3 - }, - } - mb := &block.MiniBlock{SenderShardID: 2, ReceiverShardID: 3, Type: block.TxBlock} err := process.CheckMiniBlock(mb, shardCoordinator) require.ErrorIs(t, err, process.ErrInvalidShardId) @@ -2423,17 +2424,6 @@ func TestCheckMiniBlock(t *testing.T) { t.Run("rewards miniblock should be from meta", func(t *testing.T) { t.Parallel() - selfShardID := uint32(1) - - shardCoordinator := &mock.ShardCoordinatorStub{ - SelfIdCalled: func() uint32 { - return selfShardID - }, - NumberOfShardsCalled: func() uint32 { - return 3 - }, - } - mb := &block.MiniBlock{ SenderShardID: core.MetachainShardId, ReceiverShardID: selfShardID, @@ -2526,12 +2516,6 @@ func TestCheckMiniBlock(t *testing.T) { t.Run("nil tx hash, should fail", func(t *testing.T) { t.Parallel() - shardCoordinator := &mock.ShardCoordinatorStub{ - NumberOfShardsCalled: func() uint32 { - return 3 - }, - } - mb := &block.MiniBlock{ SenderShardID: shardCoordinator.SelfId(), ReceiverShardID: 1, Type: block.TxBlock, @@ -2545,12 +2529,6 @@ func TestCheckMiniBlock(t *testing.T) { t.Run("invalid reserved field, should fail", func(t *testing.T) { t.Parallel() - shardCoordinator := &mock.ShardCoordinatorStub{ - NumberOfShardsCalled: func() uint32 { - return 3 - }, - } - mb := &block.MiniBlock{ SenderShardID: shardCoordinator.SelfId(), ReceiverShardID: 1, Type: block.TxBlock, @@ -2561,19 +2539,108 @@ func TestCheckMiniBlock(t *testing.T) { require.ErrorIs(t, err, process.ErrReservedFieldInvalid) }) - t.Run("should work", func(t *testing.T) { + t.Run("tx block should be from shards", func(t *testing.T) { t.Parallel() - selfShardID := uint32(1) + mb := &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: selfShardID, + Type: block.TxBlock, + } + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) - shardCoordinator := &mock.ShardCoordinatorStub{ - SelfIdCalled: func() uint32 { - return selfShardID - }, - NumberOfShardsCalled: func() uint32 { - return 3 - }, + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: 2, + Type: block.TxBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + }) + + t.Run("scr block should be to specific shard", func(t *testing.T) { + t.Parallel() + + mb := &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: core.AllShardId, + Type: block.SmartContractResultBlock, + } + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: 2, + Type: block.SmartContractResultBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: core.MetachainShardId, + Type: block.SmartContractResultBlock, } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + }) + + t.Run("invalid and receipts blocks must be intra shard", func(t *testing.T) { + t.Parallel() + + mb := &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: selfShardID, + Type: block.ReceiptBlock, + } + err := process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: 2, + Type: block.ReceiptBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: selfShardID, + Type: block.ReceiptBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: core.MetachainShardId, + ReceiverShardID: selfShardID, + Type: block.InvalidBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: 2, + Type: block.InvalidBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: selfShardID, + Type: block.InvalidBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + }) + + t.Run("should work", func(t *testing.T) { + t.Parallel() mb := &block.MiniBlock{ SenderShardID: 2, @@ -2592,9 +2659,25 @@ func TestCheckMiniBlock(t *testing.T) { require.Nil(t, err) mb = &block.MiniBlock{ - SenderShardID: core.MetachainShardId, + SenderShardID: selfShardID, ReceiverShardID: selfShardID, - Type: block.TxBlock, + Type: block.InvalidBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: selfShardID, + Type: block.ReceiptBlock, + } + err = process.CheckMiniBlock(mb, shardCoordinator) + require.Nil(t, err) + + mb = &block.MiniBlock{ + SenderShardID: selfShardID, + ReceiverShardID: core.MetachainShardId, + Type: block.SmartContractResultBlock, } err = process.CheckMiniBlock(mb, shardCoordinator) require.Nil(t, err) From d60a04003786701d098540665b5256bcd379b3e3 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 7 May 2026 12:54:40 +0300 Subject: [PATCH 028/116] refactor tx immunization --- consensus/spos/worker.go | 10 ++ consensus/spos/worker_internal_test.go | 93 ++++++++++ consensus/spos/worker_test.go | 1 + dataRetriever/interface.go | 3 +- dataRetriever/shardedData/interface.go | 3 +- dataRetriever/shardedData/shardedData.go | 18 +- dataRetriever/shardedData/shardedData_test.go | 3 +- dataRetriever/txpool/interface.go | 3 +- dataRetriever/txpool/shardedTxPool.go | 12 +- dataRetriever/txpool/shardedTxPool_test.go | 3 +- factory/consensus/consensusComponents.go | 1 + factory/consensus/consensusComponents_test.go | 1 + factory/processing/processComponents.go | 1 + process/track/miniBlockTrack.go | 161 +++++++++++++++++- process/track/miniBlockTrack_test.go | 122 ++++++++++--- testscommon/shardedDataCacheNotifierMock.go | 6 +- testscommon/shardedDataStub.go | 14 +- testscommon/txcachemocks/txCacheMock.go | 14 +- 18 files changed, 423 insertions(+), 46 deletions(-) create mode 100644 consensus/spos/worker_internal_test.go diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 86cd3b39dfa..28fe204f0fa 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -82,6 +82,7 @@ type Worker struct { antifloodHandler consensus.P2PAntifloodHandler poolAdder PoolAdder + whiteListHandler process.WhiteListHandler cancelFunc func() consensusMessageValidator *consensusMessageValidator @@ -114,6 +115,7 @@ type WorkerArgs struct { NetworkShardingCollector consensus.NetworkShardingCollector AntifloodHandler consensus.P2PAntifloodHandler PoolAdder PoolAdder + WhiteListHandler process.WhiteListHandler SignatureSize int PublicKeySize int AppStatusHandler core.AppStatusHandler @@ -169,6 +171,7 @@ func NewWorker(args *WorkerArgs) (*Worker, error) { networkShardingCollector: args.NetworkShardingCollector, antifloodHandler: args.AntifloodHandler, poolAdder: args.PoolAdder, + whiteListHandler: args.WhiteListHandler, nodeRedundancyHandler: args.NodeRedundancyHandler, peerBlacklistHandler: args.PeerBlacklistHandler, closer: closing.NewSafeChanCloser(), @@ -265,6 +268,9 @@ func checkNewWorkerParams(args *WorkerArgs) error { if check.IfNil(args.PoolAdder) { return ErrNilPoolAdder } + if check.IfNil(args.WhiteListHandler) { + return process.ErrNilWhiteListHandler + } if check.IfNil(args.AppStatusHandler) { return ErrNilAppStatusHandler } @@ -688,6 +694,10 @@ func (wrk *Worker) addBlockToPool(bodyBytes []byte) { if err != nil { return } + if miniblock.SenderShardID != wrk.shardCoordinator.SelfId() && + !wrk.whiteListHandler.IsWhiteListedAtLeastOne([][]byte{hash}) { + continue + } wrk.poolAdder.Put(hash, miniblock, miniblock.Size()) } } diff --git a/consensus/spos/worker_internal_test.go b/consensus/spos/worker_internal_test.go new file mode 100644 index 00000000000..dc3fca3c0da --- /dev/null +++ b/consensus/spos/worker_internal_test.go @@ -0,0 +1,93 @@ +package spos + +import ( + "bytes" + "testing" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/stretchr/testify/require" + + consensusMock "github.com/multiversx/mx-chain-go/consensus/mock" + "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/cache" + "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" +) + +func TestWorker_AddBlockToPoolSkipsNonWhitelistedCrossShardMiniBlocks(t *testing.T) { + t.Parallel() + + miniBlock := &block.MiniBlock{ + SenderShardID: 1, + ReceiverShardID: 0, + Type: block.TxBlock, + TxHashes: [][]byte{[]byte("tx-hash")}, + } + + putCalled := false + worker := &Worker{ + blockProcessor: &testscommon.BlockProcessorStub{ + DecodeBlockBodyCalled: func(_ []byte) data.BodyHandler { + return &block.Body{MiniBlocks: []*block.MiniBlock{miniBlock}} + }, + }, + marshalizer: &consensusMock.MarshalizerMock{}, + hasher: &hashingMocks.HasherMock{}, + shardCoordinator: testscommon.NewMultiShardsCoordinatorMock(2), + whiteListHandler: &testscommon.WhiteListHandlerStub{}, + poolAdder: &cache.CacherStub{ + PutCalled: func(key []byte, value interface{}, sizeInBytes int) (evicted bool) { + putCalled = true + return false + }, + }, + } + + worker.addBlockToPool([]byte("body")) + + require.False(t, putCalled) +} + +func TestWorker_AddBlockToPoolAcceptsWhitelistedCrossShardMiniBlocks(t *testing.T) { + t.Parallel() + + miniBlock := &block.MiniBlock{ + SenderShardID: 1, + ReceiverShardID: 0, + Type: block.TxBlock, + TxHashes: [][]byte{[]byte("tx-hash")}, + } + marshalizer := &consensusMock.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} + expectedHash, err := core.CalculateHash(marshalizer, hasher, miniBlock) + require.NoError(t, err) + + putCalled := false + worker := &Worker{ + blockProcessor: &testscommon.BlockProcessorStub{ + DecodeBlockBodyCalled: func(_ []byte) data.BodyHandler { + return &block.Body{MiniBlocks: []*block.MiniBlock{miniBlock}} + }, + }, + marshalizer: marshalizer, + hasher: hasher, + shardCoordinator: testscommon.NewMultiShardsCoordinatorMock(2), + whiteListHandler: &testscommon.WhiteListHandlerStub{ + IsWhiteListedAtLeastOneCalled: func(identifiers [][]byte) bool { + return len(identifiers) == 1 && bytes.Equal(identifiers[0], expectedHash) + }, + }, + poolAdder: &cache.CacherStub{ + PutCalled: func(key []byte, value interface{}, sizeInBytes int) (evicted bool) { + putCalled = true + require.True(t, bytes.Equal(expectedHash, key)) + return false + }, + }, + } + + worker.addBlockToPool([]byte("body")) + + require.True(t, putCalled) +} diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index a144b88dcff..dbbf3178f3e 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -117,6 +117,7 @@ func createDefaultWorkerArgs(appStatusHandler core.AppStatusHandler) *spos.Worke NetworkShardingCollector: &p2pmocks.NetworkShardingCollectorStub{}, AntifloodHandler: createMockP2PAntifloodHandler(), PoolAdder: poolAdder, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, SignatureSize: SignatureSize, PublicKeySize: PublicKeySize, AppStatusHandler: appStatusHandler, diff --git a/dataRetriever/interface.go b/dataRetriever/interface.go index f9d68fa9f92..d4ed66847dd 100644 --- a/dataRetriever/interface.go +++ b/dataRetriever/interface.go @@ -174,7 +174,8 @@ type ShardedDataCacherNotifier interface { SearchFirstData(key []byte) (value interface{}, ok bool) RemoveData(key []byte, cacheId string) RemoveSetOfDataFromPool(keys [][]byte, cacheId string) - ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheId string) + ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheId string, nonce uint64) + SetOldestImmuneNonce(cacheId string, nonce uint64) RemoveDataFromAllShards(key []byte) MergeShardStores(sourceCacheID, destCacheID string) Clear() diff --git a/dataRetriever/shardedData/interface.go b/dataRetriever/shardedData/interface.go index 75fd0181094..fa9f845d393 100644 --- a/dataRetriever/shardedData/interface.go +++ b/dataRetriever/shardedData/interface.go @@ -6,7 +6,8 @@ import ( type immunityCache interface { storage.Cacher - ImmunizeKeys(keys [][]byte) (numNowTotal, numFutureTotal int) + ImmunizeKeys(keys [][]byte, nonce uint64) (numNowTotal, numFutureTotal int) + SetOldestImmuneNonce(nonce uint64) RemoveWithResult(key []byte) bool NumBytes() int Diagnose(deep bool) diff --git a/dataRetriever/shardedData/shardedData.go b/dataRetriever/shardedData/shardedData.go index 0724473d07b..dd9d4a6acdb 100644 --- a/dataRetriever/shardedData/shardedData.go +++ b/dataRetriever/shardedData/shardedData.go @@ -183,11 +183,21 @@ func (sd *shardedData) RemoveSetOfDataFromPool(keys [][]byte, cacheID string) { ) } -// ImmunizeSetOfDataAgainstEviction marks the items as non-evictable -func (sd *shardedData) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheID string) { +// ImmunizeSetOfDataAgainstEviction marks the items as non-evictable for the provided confirmation nonce +func (sd *shardedData) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheID string, nonce uint64) { store := sd.getOrCreateShardStoreWithLock(cacheID) - numNow, numFuture := store.cache.ImmunizeKeys(keys) - log.Trace("shardedData.ImmunizeSetOfDataAgainstEviction()", "name", sd.name, "cacheID", cacheID, "len(keys)", len(keys), "numNow", numNow, "numFuture", numFuture) + numNow, numFuture := store.cache.ImmunizeKeys(keys, nonce) + log.Trace("shardedData.ImmunizeSetOfDataAgainstEviction()", "name", sd.name, "cacheID", cacheID, "len(keys)", len(keys), "numNow", numNow, "numFuture", numFuture, "nonce", nonce) +} + +// SetOldestImmuneNonce deactivates immunity below the provided nonce +func (sd *shardedData) SetOldestImmuneNonce(cacheID string, nonce uint64) { + store := sd.shardStore(cacheID) + if store == nil { + return + } + + store.cache.SetOldestImmuneNonce(nonce) } // RemoveData will remove data hash from the corresponding shard store diff --git a/dataRetriever/shardedData/shardedData_test.go b/dataRetriever/shardedData/shardedData_test.go index d9ab827df10..96434a2652f 100644 --- a/dataRetriever/shardedData/shardedData_test.go +++ b/dataRetriever/shardedData/shardedData_test.go @@ -330,7 +330,8 @@ func TestShardedData_ImmunizeSetOfDataAgainstEviction(t *testing.T) { t.Parallel() sd, _ := NewShardedData("", defaultTestConfig) - sd.ImmunizeSetOfDataAgainstEviction([][]byte{[]byte("aaa")}, "0") + sd.ImmunizeSetOfDataAgainstEviction([][]byte{[]byte("aaa")}, "0", 7) + sd.SetOldestImmuneNonce("0", 7) } func TestShardedData_GetCounts(t *testing.T) { diff --git a/dataRetriever/txpool/interface.go b/dataRetriever/txpool/interface.go index ee55a246a48..5242c5d1b57 100644 --- a/dataRetriever/txpool/interface.go +++ b/dataRetriever/txpool/interface.go @@ -14,7 +14,8 @@ type txCache interface { AddTx(tx *txcache.WrappedTransaction) (ok bool, added bool) GetByTxHash(txHash []byte) (*txcache.WrappedTransaction, bool) RemoveTxByHash(txHash []byte) bool - ImmunizeTxsAgainstEviction(keys [][]byte) + ImmunizeTxsAgainstEviction(keys [][]byte, nonce uint64) + SetOldestImmuneNonce(nonce uint64) ForEachTransaction(function txcache.ForEachTransaction) NumBytes() int Diagnose(deep bool) diff --git a/dataRetriever/txpool/shardedTxPool.go b/dataRetriever/txpool/shardedTxPool.go index 0f40817893d..2759765c3c9 100644 --- a/dataRetriever/txpool/shardedTxPool.go +++ b/dataRetriever/txpool/shardedTxPool.go @@ -162,10 +162,16 @@ func (txPool *shardedTxPool) createTxCache(cacheID string) txCache { return cache } -// ImmunizeSetOfDataAgainstEviction marks the items as non-evictable -func (txPool *shardedTxPool) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheID string) { +// ImmunizeSetOfDataAgainstEviction marks the items as non-evictable for the provided confirmation nonce +func (txPool *shardedTxPool) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheID string, nonce uint64) { shard := txPool.getOrCreateShard(cacheID) - shard.Cache.ImmunizeTxsAgainstEviction(keys) + shard.Cache.ImmunizeTxsAgainstEviction(keys, nonce) +} + +// SetOldestImmuneNonce deactivates immunity below the provided nonce +func (txPool *shardedTxPool) SetOldestImmuneNonce(cacheID string, nonce uint64) { + shard := txPool.getOrCreateShard(cacheID) + shard.Cache.SetOldestImmuneNonce(nonce) } // AddData adds the transaction to the cache diff --git a/dataRetriever/txpool/shardedTxPool_test.go b/dataRetriever/txpool/shardedTxPool_test.go index 1b3ab585dc3..d503c269252 100644 --- a/dataRetriever/txpool/shardedTxPool_test.go +++ b/dataRetriever/txpool/shardedTxPool_test.go @@ -358,7 +358,8 @@ func TestShardedTxPool_ImmunizeSetOfDataAgainstEviction(t *testing.T) { poolAsInterface, _ := newTxPoolToTest() pool := poolAsInterface.(*shardedTxPool) - pool.ImmunizeSetOfDataAgainstEviction([][]byte{[]byte("hash")}, "0") + pool.ImmunizeSetOfDataAgainstEviction([][]byte{[]byte("hash")}, "0", 7) + pool.SetOldestImmuneNonce("0", 7) } func Test_IsInterfaceNil(t *testing.T) { diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index 39efa2c4240..20688572df0 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -216,6 +216,7 @@ func (ccf *consensusComponentsFactory) Create() (*consensusComponents, error) { NetworkShardingCollector: ccf.processComponents.PeerShardMapper(), AntifloodHandler: ccf.networkComponents.InputAntiFloodHandler(), PoolAdder: ccf.dataComponents.Datapool().MiniBlocks(), + WhiteListHandler: ccf.processComponents.WhiteListHandler(), SignatureSize: ccf.config.ValidatorPubkeyConverter.SignatureLength, PublicKeySize: ccf.config.ValidatorPubkeyConverter.Length, AppStatusHandler: ccf.statusCoreComponents.AppStatusHandler(), diff --git a/factory/consensus/consensusComponents_test.go b/factory/consensus/consensusComponents_test.go index 161c49e777f..d8916fc4f89 100644 --- a/factory/consensus/consensusComponents_test.go +++ b/factory/consensus/consensusComponents_test.go @@ -146,6 +146,7 @@ func createMockConsensusComponentsFactoryArgs() consensusComp.ConsensusComponent HeaderSigVerif: &consensusMocks.HeaderSigVerifierMock{}, HeaderIntegrVerif: &mock.HeaderIntegrityVerifierStub{}, FallbackHdrValidator: &testscommon.FallBackHeaderValidatorStub{}, + WhiteListHandlerInternal: &testscommon.WhiteListHandlerStub{}, SentSignaturesTrackerInternal: &testscommon.SentSignatureTrackerStub{}, BlockchainHookField: &testscommon.BlockChainHookStub{}, }, diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 086ea4b0546..92ac01fe17e 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -520,6 +520,7 @@ func (pcf *processComponentsFactory) Create() (*processComponents, error) { _, err = track.NewMiniBlockTrack( pcf.data.Datapool(), + blockTracker, pcf.bootstrapComponents.ShardCoordinator(), pcf.whiteListHandler, ) diff --git a/process/track/miniBlockTrack.go b/process/track/miniBlockTrack.go index 900846f67ff..6fdbdd5cf9c 100644 --- a/process/track/miniBlockTrack.go +++ b/process/track/miniBlockTrack.go @@ -1,8 +1,11 @@ package track import ( + "sync" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" @@ -10,6 +13,12 @@ import ( "github.com/multiversx/mx-chain-go/storage" ) +type confirmedMiniBlockInfo struct { + cacheID string + mbType block.Type + nonce uint64 +} + type miniBlockTrack struct { blockTransactionsPool dataRetriever.ShardedDataCacherNotifier rewardTransactionsPool dataRetriever.ShardedDataCacherNotifier @@ -17,11 +26,14 @@ type miniBlockTrack struct { miniBlocksPool storage.Cacher shardCoordinator sharding.Coordinator whitelistHandler process.WhiteListHandler + mutConfirmedMiniBlocks sync.RWMutex + confirmedMiniBlocks map[string]confirmedMiniBlockInfo } // NewMiniBlockTrack creates an object for tracking the received mini blocks func NewMiniBlockTrack( dataPool dataRetriever.PoolsHolder, + blockTracker process.BlockTracker, shardCoordinator sharding.Coordinator, whitelistHandler process.WhiteListHandler, ) (*miniBlockTrack, error) { @@ -41,6 +53,9 @@ func NewMiniBlockTrack( if check.IfNil(dataPool.MiniBlocks()) { return nil, process.ErrNilMiniBlockPool } + if check.IfNil(blockTracker) { + return nil, process.ErrNilBlockTracker + } if check.IfNil(shardCoordinator) { return nil, process.ErrNilShardCoordinator } @@ -55,9 +70,11 @@ func NewMiniBlockTrack( miniBlocksPool: dataPool.MiniBlocks(), shardCoordinator: shardCoordinator, whitelistHandler: whitelistHandler, + confirmedMiniBlocks: make(map[string]confirmedMiniBlockInfo), } mbt.miniBlocksPool.RegisterHandler(mbt.receivedMiniBlock, core.UniqueIdentifier()) + mbt.registerBlockTrackerHandlers(blockTracker) return &mbt, nil } @@ -84,16 +101,12 @@ func (mbt *miniBlockTrack) receivedMiniBlock(key []byte, value interface{}) { return } - // TODO - stop reusing miniBlock.TxHashes for peer changes, add new fields - transactionPool := mbt.getTransactionPool(miniBlock.Type) - if check.IfNil(transactionPool) { + confirmationInfo, ok := mbt.getConfirmedMiniBlockInfo(key) + if !ok { return } - mbt.whitelistHandler.Add(miniBlock.TxHashes) - - strCache := process.ShardCacherIdentifier(miniBlock.SenderShardID, miniBlock.ReceiverShardID) - transactionPool.ImmunizeSetOfDataAgainstEviction(miniBlock.TxHashes, strCache) + mbt.immunizeMiniBlock(key, miniBlock, confirmationInfo) } func (mbt *miniBlockTrack) getTransactionPool(mbType block.Type) dataRetriever.ShardedDataCacherNotifier { @@ -108,3 +121,137 @@ func (mbt *miniBlockTrack) getTransactionPool(mbType block.Type) dataRetriever.S return nil } + +func (mbt *miniBlockTrack) registerBlockTrackerHandlers(blockTracker process.BlockTracker) { + if mbt.shardCoordinator.SelfId() == core.MetachainShardId { + blockTracker.RegisterCrossNotarizedHeadersHandler(func(_ uint32, headers []data.HeaderHandler, _ [][]byte) { + mbt.registerConfirmedMiniBlocks(headers) + }) + return + } + + blockTracker.RegisterFinalMetachainHeadersHandler(func(_ uint32, headers []data.HeaderHandler, _ [][]byte) { + mbt.registerConfirmedMiniBlocks(headers) + }) +} + +func (mbt *miniBlockTrack) registerConfirmedMiniBlocks(headers []data.HeaderHandler) { + for _, header := range headers { + mbt.registerConfirmedMiniBlocksForHeader(header) + } +} + +func (mbt *miniBlockTrack) registerConfirmedMiniBlocksForHeader(header data.HeaderHandler) { + if check.IfNil(header) { + return + } + + switch typedHeader := header.(type) { + case data.MetaHeaderHandler: + mbt.registerFromMiniBlockHeaders(typedHeader.GetNonce(), core.MetachainShardId, typedHeader.GetMiniBlockHeaderHandlers()) + for _, shardInfo := range typedHeader.GetShardInfoHandlers() { + mbt.registerFromMiniBlockHeaders(typedHeader.GetNonce(), shardInfo.GetShardID(), shardInfo.GetShardMiniBlockHeaderHandlers()) + } + case data.ShardHeaderHandler: + mbt.registerFromMiniBlockHeaders(typedHeader.GetNonce(), typedHeader.GetShardID(), typedHeader.GetMiniBlockHeaderHandlers()) + } +} + +func (mbt *miniBlockTrack) registerFromMiniBlockHeaders( + nonce uint64, + processingShard uint32, + miniBlockHeaders []data.MiniBlockHeaderHandler, +) { + selfShardID := mbt.shardCoordinator.SelfId() + for _, miniBlockHeader := range miniBlockHeaders { + receiverShard := miniBlockHeader.GetReceiverShardID() + receiverIsSelfShard := receiverShard == selfShardID || (receiverShard == core.AllShardId && processingShard == core.MetachainShardId) + senderShard := miniBlockHeader.GetSenderShardID() + if !receiverIsSelfShard || senderShard == selfShardID { + continue + } + + cacheID := process.ShardCacherIdentifier(senderShard, receiverShard) + mbInfo := confirmedMiniBlockInfo{ + cacheID: cacheID, + mbType: block.Type(miniBlockHeader.GetTypeInt32()), + nonce: nonce, + } + + mbt.storeConfirmedMiniBlockInfo(miniBlockHeader.GetHash(), mbInfo) + transactionPool := mbt.getTransactionPool(mbInfo.mbType) + if check.IfNil(transactionPool) { + continue + } + + transactionPool.SetOldestImmuneNonce(cacheID, nonce) + mbt.cleanupConfirmedMiniBlocks(cacheID, nonce) + mbt.tryProcessStoredMiniBlock(miniBlockHeader.GetHash(), mbInfo) + } +} + +func (mbt *miniBlockTrack) tryProcessStoredMiniBlock(miniBlockHash []byte, confirmationInfo confirmedMiniBlockInfo) { + value, ok := mbt.miniBlocksPool.Peek(miniBlockHash) + if !ok { + return + } + + miniBlock, ok := value.(*block.MiniBlock) + if !ok { + return + } + + mbt.immunizeMiniBlock(miniBlockHash, miniBlock, confirmationInfo) +} + +func (mbt *miniBlockTrack) immunizeMiniBlock(miniBlockHash []byte, miniBlock *block.MiniBlock, confirmationInfo confirmedMiniBlockInfo) { + transactionPool := mbt.getTransactionPool(miniBlock.Type) + if check.IfNil(transactionPool) { + return + } + + mbt.whitelistHandler.Add(miniBlock.TxHashes) + transactionPool.SetOldestImmuneNonce(confirmationInfo.cacheID, confirmationInfo.nonce) + transactionPool.ImmunizeSetOfDataAgainstEviction(miniBlock.TxHashes, confirmationInfo.cacheID, confirmationInfo.nonce) + mbt.removeConfirmedMiniBlockInfo(miniBlockHash) +} + +func (mbt *miniBlockTrack) storeConfirmedMiniBlockInfo(miniBlockHash []byte, info confirmedMiniBlockInfo) { + mbt.mutConfirmedMiniBlocks.Lock() + defer mbt.mutConfirmedMiniBlocks.Unlock() + + key := string(miniBlockHash) + existingInfo, exists := mbt.confirmedMiniBlocks[key] + if exists && existingInfo.nonce >= info.nonce { + return + } + + mbt.confirmedMiniBlocks[key] = info +} + +func (mbt *miniBlockTrack) getConfirmedMiniBlockInfo(miniBlockHash []byte) (confirmedMiniBlockInfo, bool) { + mbt.mutConfirmedMiniBlocks.RLock() + defer mbt.mutConfirmedMiniBlocks.RUnlock() + + info, ok := mbt.confirmedMiniBlocks[string(miniBlockHash)] + return info, ok +} + +func (mbt *miniBlockTrack) removeConfirmedMiniBlockInfo(miniBlockHash []byte) { + mbt.mutConfirmedMiniBlocks.Lock() + delete(mbt.confirmedMiniBlocks, string(miniBlockHash)) + mbt.mutConfirmedMiniBlocks.Unlock() +} + +func (mbt *miniBlockTrack) cleanupConfirmedMiniBlocks(cacheID string, nonce uint64) { + mbt.mutConfirmedMiniBlocks.Lock() + defer mbt.mutConfirmedMiniBlocks.Unlock() + + for key, info := range mbt.confirmedMiniBlocks { + if info.cacheID != cacheID || info.nonce >= nonce { + continue + } + + delete(mbt.confirmedMiniBlocks, key) + } +} diff --git a/process/track/miniBlockTrack_test.go b/process/track/miniBlockTrack_test.go index 6a72d7ad9d0..60068660bbf 100644 --- a/process/track/miniBlockTrack_test.go +++ b/process/track/miniBlockTrack_test.go @@ -3,6 +3,8 @@ package track_test import ( "testing" + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/stretchr/testify/assert" @@ -19,7 +21,7 @@ import ( func TestNewMiniBlockTrack_NilDataPoolHolderErr(t *testing.T) { t.Parallel() - mbt, err := track.NewMiniBlockTrack(nil, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(nil, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilPoolsHolder, err) @@ -33,7 +35,7 @@ func TestNewMiniBlockTrack_NilTxsPoolErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilTransactionPool, err) @@ -50,7 +52,7 @@ func TestNewMiniBlockTrack_NilRewardTxsPoolErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilRewardTxDataPool, err) @@ -70,7 +72,7 @@ func TestNewMiniBlockTrack_NilUnsignedTxsPoolErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilUnsignedTxDataPool, err) @@ -93,17 +95,27 @@ func TestNewMiniBlockTrack_NilMiniBlockPoolShouldErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilMiniBlockPool, err) } +func TestNewMiniBlockTrack_NilBlockTrackerErr(t *testing.T) { + t.Parallel() + + dataPool := createDataPool() + miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, nil, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + + assert.Nil(t, miniBlockTrack) + assert.Equal(t, process.ErrNilBlockTracker, err) +} + func TestNewMiniBlockTrack_NilShardCoordinatorErr(t *testing.T) { t.Parallel() dataPool := createDataPool() - miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, nil, &testscommon.WhiteListHandlerStub{}) + miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), nil, &testscommon.WhiteListHandlerStub{}) assert.Nil(t, miniBlockTrack) assert.Equal(t, process.ErrNilShardCoordinator, err) @@ -113,7 +125,7 @@ func TestNewMiniBlockTrack_NilWhitelistHandlerErr(t *testing.T) { t.Parallel() dataPool := createDataPool() - miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), nil) + miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), nil) assert.Nil(t, miniBlockTrack) assert.Equal(t, process.ErrNilWhiteListHandler, err) @@ -123,7 +135,7 @@ func TestNewMiniBlockTrack_ShouldWork(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, err := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, err) assert.NotNil(t, mbt) @@ -133,11 +145,11 @@ func TestReceivedMiniBlock_ShouldReturnIfKeyIsNil(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ - ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string) { + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string, nonce uint64) { wasCalled = true }, } @@ -151,11 +163,11 @@ func TestReceivedMiniBlock_ShouldReturnIfWrongTypeAssertion(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ - ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string) { + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string, nonce uint64) { wasCalled = true }, } @@ -169,11 +181,11 @@ func TestReceivedMiniBlock_ShouldReturnIfMiniBlockIsNotCrossShardDestMe(t *testi t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ - ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string) { + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string, nonce uint64) { wasCalled = true }, } @@ -187,11 +199,11 @@ func TestReceivedMiniBlock_ShouldReturnIfMiniBlockTypeIsWrong(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ - ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string) { + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string, nonce uint64) { wasCalled = true }, } @@ -206,15 +218,15 @@ func TestReceivedMiniBlock_ShouldReturnIfMiniBlockTypeIsWrong(t *testing.T) { assert.False(t, wasCalled) } -func TestReceivedMiniBlock_ShouldWork(t *testing.T) { +func TestReceivedMiniBlock_ShouldNotImmunizeUnconfirmedMiniBlock(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ - ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string) { + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string, nonce uint64) { wasCalled = true }, } @@ -226,7 +238,73 @@ func TestReceivedMiniBlock_ShouldWork(t *testing.T) { Type: block.TxBlock, }) + assert.False(t, wasCalled) +} + +func TestReceivedMiniBlock_ShouldImmunizeConfirmedMiniBlock(t *testing.T) { + t.Parallel() + + dataPool := createDataPool() + blockTracker := createBlockTracker() + + var finalMetachainHeadersHandler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte) + blockTracker.RegisterFinalMetachainHeadersHandlerCalled = func(handler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte)) { + finalMetachainHeadersHandler = handler + } + + whitelistCalled := false + whiteListHandler := &testscommon.WhiteListHandlerStub{ + AddCalled: func(keys [][]byte) { + whitelistCalled = true + }, + } + mbt, _ := track.NewMiniBlockTrack(dataPool, blockTracker, mock.NewMultipleShardsCoordinatorMock(), whiteListHandler) + + var cacheID string + var nonce uint64 + wasCalled := false + blockTransactionsPool := &testscommon.ShardedDataStub{ + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheId string, providedNonce uint64) { + wasCalled = true + cacheID = destCacheId + nonce = providedNonce + }, + } + mbt.SetBlockTransactionsPool(blockTransactionsPool) + + finalMetachainHeadersHandler(core.MetachainShardId, []data.HeaderHandler{ + &block.MetaBlock{ + Nonce: 7, + ShardInfo: []block.ShardData{ + { + ShardID: 1, + ShardMiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: []byte("mb_hash"), + SenderShardID: 1, + ReceiverShardID: 0, + Type: block.TxBlock, + }, + }, + }, + }, + }, + }, nil) + + mbt.ReceivedMiniBlock( + []byte("mb_hash"), + &block.MiniBlock{ + SenderShardID: 1, + ReceiverShardID: 0, + Type: block.TxBlock, + TxHashes: [][]byte{[]byte("txHash")}, + }, + ) + assert.True(t, wasCalled) + assert.True(t, whitelistCalled) + assert.Equal(t, process.ShardCacherIdentifier(1, 0), cacheID) + assert.Equal(t, uint64(7), nonce) } func TestGetTransactionPool_ShouldWork(t *testing.T) { @@ -261,7 +339,7 @@ func TestGetTransactionPool_ShouldWork(t *testing.T) { return cache.NewCacherStub() }, } - mbt, _ := track.NewMiniBlockTrack(dataPool, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) tp := mbt.GetTransactionPool(block.TxBlock) assert.Equal(t, blockTransactionsPool, tp) @@ -292,3 +370,7 @@ func createDataPool() dataRetriever.PoolsHolder { }, } } + +func createBlockTracker() *mock.BlockTrackerMock { + return &mock.BlockTrackerMock{} +} diff --git a/testscommon/shardedDataCacheNotifierMock.go b/testscommon/shardedDataCacheNotifierMock.go index f6043415b08..f8a341c5b8b 100644 --- a/testscommon/shardedDataCacheNotifierMock.go +++ b/testscommon/shardedDataCacheNotifierMock.go @@ -76,7 +76,11 @@ func (mock *ShardedDataCacheNotifierMock) RemoveSetOfDataFromPool(keys [][]byte, } // ImmunizeSetOfDataAgainstEviction - -func (mock *ShardedDataCacheNotifierMock) ImmunizeSetOfDataAgainstEviction(_ [][]byte, _ string) { +func (mock *ShardedDataCacheNotifierMock) ImmunizeSetOfDataAgainstEviction(_ [][]byte, _ string, _ uint64) { +} + +// SetOldestImmuneNonce - +func (mock *ShardedDataCacheNotifierMock) SetOldestImmuneNonce(_ string, _ uint64) { } // RemoveDataFromAllShards - diff --git a/testscommon/shardedDataStub.go b/testscommon/shardedDataStub.go index 2a082afe96f..cba3aa8e40a 100644 --- a/testscommon/shardedDataStub.go +++ b/testscommon/shardedDataStub.go @@ -18,7 +18,8 @@ type ShardedDataStub struct { ClearCalled func() ClearShardStoreCalled func(cacheID string) RemoveSetOfDataFromPoolCalled func(keys [][]byte, destCacheID string) - ImmunizeSetOfDataAgainstEvictionCalled func(keys [][]byte, cacheID string) + ImmunizeSetOfDataAgainstEvictionCalled func(keys [][]byte, cacheID string, nonce uint64) + SetOldestImmuneNonceCalled func(cacheID string, nonce uint64) CreateShardStoreCalled func(destCacheID string) GetCountsCalled func() counting.CountsWithSize KeysCalled func() [][]byte @@ -102,9 +103,16 @@ func (sd *ShardedDataStub) RemoveSetOfDataFromPool(keys [][]byte, cacheID string } // ImmunizeSetOfDataAgainstEviction - -func (sd *ShardedDataStub) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheID string) { +func (sd *ShardedDataStub) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheID string, nonce uint64) { if sd.ImmunizeSetOfDataAgainstEvictionCalled != nil { - sd.ImmunizeSetOfDataAgainstEvictionCalled(keys, cacheID) + sd.ImmunizeSetOfDataAgainstEvictionCalled(keys, cacheID, nonce) + } +} + +// SetOldestImmuneNonce - +func (sd *ShardedDataStub) SetOldestImmuneNonce(cacheID string, nonce uint64) { + if sd.SetOldestImmuneNonceCalled != nil { + sd.SetOldestImmuneNonceCalled(cacheID, nonce) } } diff --git a/testscommon/txcachemocks/txCacheMock.go b/testscommon/txcachemocks/txCacheMock.go index c34db2d53b0..095d205d3c9 100644 --- a/testscommon/txcachemocks/txCacheMock.go +++ b/testscommon/txcachemocks/txCacheMock.go @@ -22,7 +22,8 @@ type TxCacheMock struct { AddTxCalled func(tx *txcache.WrappedTransaction) (ok bool, added bool) GetByTxHashCalled func(txHash []byte) (*txcache.WrappedTransaction, bool) RemoveTxByHashCalled func(txHash []byte) bool - ImmunizeTxsAgainstEvictionCalled func(keys [][]byte) + ImmunizeTxsAgainstEvictionCalled func(keys [][]byte, nonce uint64) + SetOldestImmuneNonceCalled func(nonce uint64) ForEachTransactionCalled func(txcache.ForEachTransaction) NumBytesCalled func() int DiagnoseCalled func(deep bool) @@ -176,9 +177,16 @@ func (cache *TxCacheMock) RemoveTxByHash(txHash []byte) bool { } // ImmunizeTxsAgainstEviction - -func (cache *TxCacheMock) ImmunizeTxsAgainstEviction(keys [][]byte) { +func (cache *TxCacheMock) ImmunizeTxsAgainstEviction(keys [][]byte, nonce uint64) { if cache.ImmunizeTxsAgainstEvictionCalled != nil { - cache.ImmunizeTxsAgainstEvictionCalled(keys) + cache.ImmunizeTxsAgainstEvictionCalled(keys, nonce) + } +} + +// SetOldestImmuneNonce - +func (cache *TxCacheMock) SetOldestImmuneNonce(nonce uint64) { + if cache.SetOldestImmuneNonceCalled != nil { + cache.SetOldestImmuneNonceCalled(nonce) } } From 6929c28daee58be7544d4421e10e05565e676cb2 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 7 May 2026 12:55:30 +0300 Subject: [PATCH 029/116] fix failing test --- integrationTests/testProcessorNode.go | 1 + process/track/miniBlockTrack.go | 1 + process/track/miniBlockTrack_test.go | 34 ++++++++++++--------------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 3344722fb65..9151c87a2a0 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -3479,6 +3479,7 @@ func GetDefaultProcessComponents() *mock.ProcessComponentsStub { CurrentEpochProviderInternal: &testscommon.CurrentEpochProviderStub{}, HistoryRepositoryInternal: &dblookupextMock.HistoryRepositoryStub{}, HardforkTriggerField: &testscommon.HardforkTriggerStub{}, + WhiteListHandlerInternal: &testscommon.WhiteListHandlerStub{}, } } diff --git a/process/track/miniBlockTrack.go b/process/track/miniBlockTrack.go index 6fdbdd5cf9c..2def2690745 100644 --- a/process/track/miniBlockTrack.go +++ b/process/track/miniBlockTrack.go @@ -205,6 +205,7 @@ func (mbt *miniBlockTrack) tryProcessStoredMiniBlock(miniBlockHash []byte, confi } func (mbt *miniBlockTrack) immunizeMiniBlock(miniBlockHash []byte, miniBlock *block.MiniBlock, confirmationInfo confirmedMiniBlockInfo) { + // TODO - stop reusing miniBlock.TxHashes for peer changes, add new fields transactionPool := mbt.getTransactionPool(miniBlock.Type) if check.IfNil(transactionPool) { return diff --git a/process/track/miniBlockTrack_test.go b/process/track/miniBlockTrack_test.go index 60068660bbf..a9dadd3c07d 100644 --- a/process/track/miniBlockTrack_test.go +++ b/process/track/miniBlockTrack_test.go @@ -21,7 +21,7 @@ import ( func TestNewMiniBlockTrack_NilDataPoolHolderErr(t *testing.T) { t.Parallel() - mbt, err := track.NewMiniBlockTrack(nil, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(nil, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilPoolsHolder, err) @@ -35,7 +35,7 @@ func TestNewMiniBlockTrack_NilTxsPoolErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilTransactionPool, err) @@ -52,7 +52,7 @@ func TestNewMiniBlockTrack_NilRewardTxsPoolErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilRewardTxDataPool, err) @@ -72,7 +72,7 @@ func TestNewMiniBlockTrack_NilUnsignedTxsPoolErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilUnsignedTxDataPool, err) @@ -95,7 +95,7 @@ func TestNewMiniBlockTrack_NilMiniBlockPoolShouldErr(t *testing.T) { return nil }, } - mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, mbt) assert.Equal(t, process.ErrNilMiniBlockPool, err) @@ -115,7 +115,7 @@ func TestNewMiniBlockTrack_NilShardCoordinatorErr(t *testing.T) { t.Parallel() dataPool := createDataPool() - miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), nil, &testscommon.WhiteListHandlerStub{}) + miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, nil, &testscommon.WhiteListHandlerStub{}) assert.Nil(t, miniBlockTrack) assert.Equal(t, process.ErrNilShardCoordinator, err) @@ -125,7 +125,7 @@ func TestNewMiniBlockTrack_NilWhitelistHandlerErr(t *testing.T) { t.Parallel() dataPool := createDataPool() - miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), nil) + miniBlockTrack, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), nil) assert.Nil(t, miniBlockTrack) assert.Equal(t, process.ErrNilWhiteListHandler, err) @@ -135,7 +135,7 @@ func TestNewMiniBlockTrack_ShouldWork(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, err := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, err := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) assert.Nil(t, err) assert.NotNil(t, mbt) @@ -145,7 +145,7 @@ func TestReceivedMiniBlock_ShouldReturnIfKeyIsNil(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ @@ -163,7 +163,7 @@ func TestReceivedMiniBlock_ShouldReturnIfWrongTypeAssertion(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ @@ -181,7 +181,7 @@ func TestReceivedMiniBlock_ShouldReturnIfMiniBlockIsNotCrossShardDestMe(t *testi t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ @@ -199,7 +199,7 @@ func TestReceivedMiniBlock_ShouldReturnIfMiniBlockTypeIsWrong(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ @@ -222,7 +222,7 @@ func TestReceivedMiniBlock_ShouldNotImmunizeUnconfirmedMiniBlock(t *testing.T) { t.Parallel() dataPool := createDataPool() - mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) wasCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ @@ -245,7 +245,7 @@ func TestReceivedMiniBlock_ShouldImmunizeConfirmedMiniBlock(t *testing.T) { t.Parallel() dataPool := createDataPool() - blockTracker := createBlockTracker() + blockTracker := &mock.BlockTrackerMock{} var finalMetachainHeadersHandler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte) blockTracker.RegisterFinalMetachainHeadersHandlerCalled = func(handler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte)) { @@ -339,7 +339,7 @@ func TestGetTransactionPool_ShouldWork(t *testing.T) { return cache.NewCacherStub() }, } - mbt, _ := track.NewMiniBlockTrack(dataPool, createBlockTracker(), mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + mbt, _ := track.NewMiniBlockTrack(dataPool, &mock.BlockTrackerMock{}, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) tp := mbt.GetTransactionPool(block.TxBlock) assert.Equal(t, blockTransactionsPool, tp) @@ -370,7 +370,3 @@ func createDataPool() dataRetriever.PoolsHolder { }, } } - -func createBlockTracker() *mock.BlockTrackerMock { - return &mock.BlockTrackerMock{} -} From 2d264527630465d35fd31aeb913d3d9b7b8d31b3 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 11 May 2026 16:14:27 +0300 Subject: [PATCH 030/116] fixes after review --- consensus/spos/worker.go | 1 + process/track/miniBlockTrack.go | 12 +++- process/track/miniBlockTrack_test.go | 95 ++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 3 deletions(-) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 28fe204f0fa..b5e90c739ff 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -696,6 +696,7 @@ func (wrk *Worker) addBlockToPool(bodyBytes []byte) { } if miniblock.SenderShardID != wrk.shardCoordinator.SelfId() && !wrk.whiteListHandler.IsWhiteListedAtLeastOne([][]byte{hash}) { + log.Trace("addBlockToPool: skipping non-whitelisted cross-shard mini block", "hash", hash) continue } wrk.poolAdder.Put(hash, miniblock, miniblock.Size()) diff --git a/process/track/miniBlockTrack.go b/process/track/miniBlockTrack.go index 2def2690745..27aaa52ab27 100644 --- a/process/track/miniBlockTrack.go +++ b/process/track/miniBlockTrack.go @@ -165,9 +165,15 @@ func (mbt *miniBlockTrack) registerFromMiniBlockHeaders( selfShardID := mbt.shardCoordinator.SelfId() for _, miniBlockHeader := range miniBlockHeaders { receiverShard := miniBlockHeader.GetReceiverShardID() - receiverIsSelfShard := receiverShard == selfShardID || (receiverShard == core.AllShardId && processingShard == core.MetachainShardId) + receiverIsAllShardsMiniBlockFromMetaHeader := receiverShard == core.AllShardId && processingShard == core.MetachainShardId + receiverIsRelevantForCurrentShard := receiverShard == selfShardID || receiverIsAllShardsMiniBlockFromMetaHeader senderShard := miniBlockHeader.GetSenderShardID() - if !receiverIsSelfShard || senderShard == selfShardID { + senderIsSelfShard := senderShard == selfShardID + // Track only miniblocks that are relevant for this shard and come from another shard. + // This includes direct cross-shard miniblocks addressed to this shard and the + // special metachain-header case where the receiver is AllShardId. + // Intra-shard miniblocks are produced and processed locally, so they are skipped here. + if !receiverIsRelevantForCurrentShard || senderIsSelfShard { continue } @@ -178,12 +184,12 @@ func (mbt *miniBlockTrack) registerFromMiniBlockHeaders( nonce: nonce, } - mbt.storeConfirmedMiniBlockInfo(miniBlockHeader.GetHash(), mbInfo) transactionPool := mbt.getTransactionPool(mbInfo.mbType) if check.IfNil(transactionPool) { continue } + mbt.storeConfirmedMiniBlockInfo(miniBlockHeader.GetHash(), mbInfo) transactionPool.SetOldestImmuneNonce(cacheID, nonce) mbt.cleanupConfirmedMiniBlocks(cacheID, nonce) mbt.tryProcessStoredMiniBlock(miniBlockHeader.GetHash(), mbInfo) diff --git a/process/track/miniBlockTrack_test.go b/process/track/miniBlockTrack_test.go index a9dadd3c07d..4fc384d3239 100644 --- a/process/track/miniBlockTrack_test.go +++ b/process/track/miniBlockTrack_test.go @@ -370,3 +370,98 @@ func createDataPool() dataRetriever.PoolsHolder { }, } } + +func TestRegisterConfirmedMiniBlocksForHeader_ShouldImmunizeStoredMiniBlock(t *testing.T) { + t.Parallel() + + miniBlockHash := []byte("mb_hash") + txHashes := [][]byte{[]byte("txHash")} + storedMiniBlock := &block.MiniBlock{ + SenderShardID: 1, + ReceiverShardID: 0, + Type: block.TxBlock, + TxHashes: txHashes, + } + + miniBlocksPool := cache.NewCacherStub() + miniBlocksPool.PeekCalled = func(key []byte) (value interface{}, ok bool) { + if string(key) != string(miniBlockHash) { + return nil, false + } + + return storedMiniBlock, true + } + + dataPool := &dataRetrieverMock.PoolsHolderStub{ + TransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return testscommon.NewShardedDataStub() + }, + RewardTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return testscommon.NewShardedDataStub() + }, + UnsignedTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return testscommon.NewShardedDataStub() + }, + MiniBlocksCalled: func() storage.Cacher { + return miniBlocksPool + }, + } + + blockTracker := &mock.BlockTrackerMock{} + var finalMetachainHeadersHandler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte) + blockTracker.RegisterFinalMetachainHeadersHandlerCalled = func(handler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte)) { + finalMetachainHeadersHandler = handler + } + + var whitelistedKeys [][]byte + whiteListHandler := &testscommon.WhiteListHandlerStub{ + AddCalled: func(keys [][]byte) { + whitelistedKeys = keys + }, + } + var immunizedKeys [][]byte + var setOldestImmuneNonceCacheID string + var immunizedCacheID string + var setOldestImmuneNonceNonce uint64 + var immunizedNonce uint64 + blockTransactionsPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceCalled: func(cacheID string, nonce uint64) { + setOldestImmuneNonceCacheID = cacheID + setOldestImmuneNonceNonce = nonce + }, + ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheID string, nonce uint64) { + immunizedKeys = keys + immunizedCacheID = destCacheID + immunizedNonce = nonce + }, + } + + mbt, _ := track.NewMiniBlockTrack(dataPool, blockTracker, mock.NewMultipleShardsCoordinatorMock(), whiteListHandler) + mbt.SetBlockTransactionsPool(blockTransactionsPool) + + finalMetachainHeadersHandler(core.MetachainShardId, []data.HeaderHandler{ + &block.MetaBlock{ + Nonce: 7, + ShardInfo: []block.ShardData{ + { + ShardID: 1, + ShardMiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: miniBlockHash, + SenderShardID: 1, + ReceiverShardID: 0, + Type: block.TxBlock, + }, + }, + }, + }, + }, + }, nil) + + assert.Equal(t, txHashes, whitelistedKeys) + assert.Equal(t, txHashes, immunizedKeys) + assert.Equal(t, process.ShardCacherIdentifier(1, 0), setOldestImmuneNonceCacheID) + assert.Equal(t, process.ShardCacherIdentifier(1, 0), immunizedCacheID) + assert.Equal(t, uint64(7), setOldestImmuneNonceNonce) + assert.Equal(t, uint64(7), immunizedNonce) +} From e6a91d5521959fcf5433e79e799decc5dd8a80ef Mon Sep 17 00:00:00 2001 From: ssd04 Date: Tue, 12 May 2026 14:45:34 +0300 Subject: [PATCH 031/116] meta mbs extra checks --- process/block/metablock.go | 15 ++ process/block/metablock_test.go | 244 ++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+) diff --git a/process/block/metablock.go b/process/block/metablock.go index 33f0b4ac917..670d3f1d4bd 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -306,6 +306,11 @@ func (mp *metaProcessor) ProcessBlock( return err } + err = mp.verifyNonEpochStartMiniBlocks(header) + if err != nil { + return err + } + mp.txCoordinator.RequestBlockTransactions(body) requestedShardHdrs, requestedFinalityAttestingShardHdrs, requestedProofs := mp.requestShardHeaders(header) @@ -555,6 +560,16 @@ func (mp *metaProcessor) verifyEpochStartMiniBlocks(metaBlock *block.MetaBlock) return nil } +func (mp *metaProcessor) verifyNonEpochStartMiniBlocks(metaBlock *block.MetaBlock) error { + for _, miniBlockHeader := range metaBlock.MiniBlockHeaders { + if miniBlockHeader.GetType() == block.RewardsBlock { + return process.ErrInvalidMiniBlockType + } + } + + return nil +} + // SetNumProcessedObj will set the num of processed headers func (mp *metaProcessor) SetNumProcessedObj(numObj uint64) { mp.headersCounter.shardMBHeadersTotalProcessed = numObj diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index ee2e2c5fd60..346e95c98f5 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -806,6 +806,250 @@ func TestMetaProcessor_ProcessBlockWithErrOnVerifyStateRootCallShouldRevertState assert.True(t, wasCalled) } +func TestMetaProcessor_ProcessBlock_MiniBlockChecks(t *testing.T) { + t.Parallel() + + hash := []byte("hash1") + miniBlock1 := &block.MiniBlock{TxHashes: [][]byte{hash}} + + txCoordinator := &testscommon.TransactionCoordinatorMock{ + CreateMbsAndProcessCrossShardTransactionsDstMeCalled: func(header data.HeaderHandler, processedMiniBlocksInfo map[string]*processedMb.ProcessedMiniBlockInfo, haveTime func() bool, haveAdditionalTime func() bool, scheduledMode bool) (slices block.MiniBlockSlice, u uint32, b bool, err error) { + return block.MiniBlockSlice{miniBlock1}, 0, true, nil + }, + } + + blkc := &testscommon.ChainHandlerStub{ + GetCurrentBlockHeaderCalled: func() data.HeaderHandler { + return &block.MetaBlock{Nonce: 0, AccumulatedFeesInEpoch: big.NewInt(0), DevFeesInEpoch: big.NewInt(0)} + }, + GetCurrentBlockHeaderHashCalled: func() []byte { + return hash + }, + GetGenesisHeaderCalled: func() data.HeaderHandler { + return &block.Header{Nonce: 0} + }, + } + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createMockComponentHolders() + coreComponents.Hash = &hashingMocks.HasherMock{} + dataComponents.BlockChain = blkc + bootstrapComponents.VersionedHdrFactory = &testscommon.VersionedHeaderFactoryStub{ + CreateCalled: func(epoch uint32) data.HeaderHandler { + return &block.MetaBlock{ + Epoch: 0, + } + }, + } + arguments := createMockMetaArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + arguments.TxCoordinator = txCoordinator + + mp, _ := blproc.NewMetaProcessor(arguments) + + t.Run("should work with valid miniblocks", func(t *testing.T) { + mb1 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash1")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.TxBlock, + } + + mbHash, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb1) + + metaBlock := &block.MetaBlock{ + Nonce: 1, + Round: 1, + PrevHash: hash, + AccumulatedFees: big.NewInt(0), + AccumulatedFeesInEpoch: big.NewInt(0), + DeveloperFees: big.NewInt(0), + DevFeesInEpoch: big.NewInt(0), + TxCount: 1, + MiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: mbHash, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.TxBlock, + TxCount: 1, + }, + }, + } + + body := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + mb1, + }, + } + + err := mp.ProcessBlock(metaBlock, body, func() time.Duration { return time.Second }) + require.Nil(t, err) + }) + + t.Run("non epoch start should not have rewards mb", func(t *testing.T) { + mb1 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash1")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.RewardsBlock, + } + + mbHash, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb1) + + metaBlock := &block.MetaBlock{ + Nonce: 1, + Round: 1, + PrevHash: hash, + AccumulatedFees: big.NewInt(0), + AccumulatedFeesInEpoch: big.NewInt(0), + DeveloperFees: big.NewInt(0), + DevFeesInEpoch: big.NewInt(0), + TxCount: 1, + MiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: mbHash, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.RewardsBlock, + TxCount: 1, + }, + }, + } + + body := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + mb1, + }, + } + + err := mp.ProcessBlock(metaBlock, body, func() time.Duration { return time.Second }) + require.Equal(t, process.ErrInvalidMiniBlockType, err) + }) + + t.Run("epoch start should have rewards or peer mb", func(t *testing.T) { + mb1 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash1")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.RewardsBlock, + } + mb2 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash2")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: core.AllShardId, + Type: block.PeerBlock, + } + + mbHash, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb1) + mbHash2, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb2) + + metaBlock := &block.MetaBlock{ + Nonce: 1, + Round: 1, + PrevHash: hash, + AccumulatedFees: big.NewInt(0), + AccumulatedFeesInEpoch: big.NewInt(0), + DeveloperFees: big.NewInt(0), + DevFeesInEpoch: big.NewInt(0), + TxCount: 1, + MiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: mbHash, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.RewardsBlock, + TxCount: 1, + }, + { + Hash: mbHash2, + SenderShardID: core.MetachainShardId, + ReceiverShardID: core.AllShardId, + Type: block.PeerBlock, + TxCount: 1, + }, + }, + EpochStart: block.EpochStart{ + LastFinalizedHeaders: []block.EpochStartShardData{ + { + ShardID: 1, + }, + }, + }, + } + + body := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + mb1, + mb2, + }, + } + + err := mp.ProcessBlock(metaBlock, body, func() time.Duration { return time.Second }) + require.Nil(t, err) + }) + + t.Run("epoch start should not have other mb types", func(t *testing.T) { + mb1 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash1")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.TxBlock, + } + mb2 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash2")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.ReceiptBlock, + } + + mbHash, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb1) + mbHash2, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb2) + + metaBlock := &block.MetaBlock{ + Nonce: 1, + Round: 1, + PrevHash: hash, + AccumulatedFees: big.NewInt(0), + AccumulatedFeesInEpoch: big.NewInt(0), + DeveloperFees: big.NewInt(0), + DevFeesInEpoch: big.NewInt(0), + TxCount: 1, + MiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: mbHash, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.TxBlock, + TxCount: 1, + }, + { + Hash: mbHash2, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.ReceiptBlock, + TxCount: 1, + }, + }, + EpochStart: block.EpochStart{ + LastFinalizedHeaders: []block.EpochStartShardData{ + { + ShardID: 1, + }, + }, + }, + } + + body := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + mb1, + mb2, + }, + } + + err := mp.ProcessBlock(metaBlock, body, func() time.Duration { return time.Second }) + require.Equal(t, process.ErrInvalidMiniBlockType, err) + }) +} + // ------- requestFinalMissingHeader func TestMetaProcessor_RequestFinalMissingHeaderShouldPass(t *testing.T) { t.Parallel() From 1aa75bfa634ab85bc3e29008951beace224c3d23 Mon Sep 17 00:00:00 2001 From: miiu Date: Tue, 12 May 2026 14:58:02 +0300 Subject: [PATCH 032/116] new version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1f3332af5a..ff7452059b8 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.9.2 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 - github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260429132446-df76d7a4bc38 + github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260512115600-4b656efb29e0 github.com/multiversx/mx-chain-vm-common-go v1.6.5 github.com/multiversx/mx-chain-vm-go v1.5.45 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 diff --git a/go.sum b/go.sum index 8472119e758..e876af0866c 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260429132446-df76d7a4bc38 h1:tInQvnDEq/fY6VomIelQd8IHBbBX9X3wBNPk1s3VMrw= -github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260429132446-df76d7a4bc38/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260512115600-4b656efb29e0 h1:S4H5Vhq2bdC6TtZuMKdcPvD+vJ2BfSxkcfvO/l43buA= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260512115600-4b656efb29e0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= From c2adf39fc5abfad5b6096301e662dab43e7d6801 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 12 May 2026 16:14:31 +0300 Subject: [PATCH 033/116] add miniblock check when consensus message received --- consensus/spos/export_test.go | 5 +++++ consensus/spos/worker.go | 8 +++++++ consensus/spos/worker_test.go | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/consensus/spos/export_test.go b/consensus/spos/export_test.go index 6ada6ceccde..3a7e053a2fd 100644 --- a/consensus/spos/export_test.go +++ b/consensus/spos/export_test.go @@ -186,6 +186,11 @@ func (wrk *Worker) SetEnableEpochsHandler(enableEpochsHandler common.EnableEpoch wrk.enableEpochsHandler = enableEpochsHandler } +// AddBlockToPool - +func (wrk *Worker) AddBlockToPool(bodyBytes []byte) { + wrk.addBlockToPool(bodyBytes) +} + // AddFutureHeaderToProcessIfNeeded - func (wrk *Worker) AddFutureHeaderToProcessIfNeeded(header data.HeaderHandler) { wrk.addFutureHeaderToProcessIfNeeded(header) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 86cd3b39dfa..9aa4db326b7 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -683,6 +683,14 @@ func (wrk *Worker) addBlockToPool(bodyBytes []byte) { return } + for _, miniblock := range body.MiniBlocks { + err := process.CheckMiniBlock(miniblock, wrk.shardCoordinator) + if err != nil { + log.Debug("addBlockToPool: invalid miniblock in received consensus body", "error", err.Error()) + return + } + } + for _, miniblock := range body.MiniBlocks { hash, err := core.CalculateHash(wrk.marshalizer, wrk.hasher, miniblock) if err != nil { diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index a144b88dcff..a1eec9d7525 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -412,6 +412,45 @@ func TestWorker_NewWorkerShouldWork(t *testing.T) { assert.False(t, check.IfNil(wrk)) } +func TestWorker_AddBlockToPoolShouldNotAddIfOneInvalidMiniBlock(t *testing.T) { + t.Parallel() + + workerArgs := createDefaultWorkerArgs(&statusHandlerMock.AppStatusHandlerStub{}) + body := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + { + Type: block.TxBlock, + SenderShardID: 0, + ReceiverShardID: 1, + }, + { + Type: block.TxBlock, + SenderShardID: 1, + ReceiverShardID: 0, + }, + // Invalid miniBlock + { + Type: block.TxBlock, + SenderShardID: 1, + ReceiverShardID: 0, + Reserved: bytes.Repeat([]byte{1}, 11), + }, + }, + } + workerArgs.BlockProcessor = &testscommon.BlockProcessorStub{ + DecodeBlockBodyCalled: func(_ []byte) data.BodyHandler { + return body + }, + } + + wrk, err := spos.NewWorker(workerArgs) + require.NoError(t, err) + + wrk.AddBlockToPool(nil) + + require.Equal(t, 0, workerArgs.PoolAdder.(*cache.CacherMock).Len()) +} + func TestWorker_ProcessReceivedMessageShouldErrIfFloodIsDetectedOnTopic(t *testing.T) { t.Parallel() From fe3d88794eeea4bc2813464c917778591358eddd Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 12 May 2026 17:39:37 +0300 Subject: [PATCH 034/116] trigger broadcast on proof received --- consensus/broadcast/delayedBroadcast.go | 141 ++++++++++++++++++++-- consensus/spos/sposFactory/sposFactory.go | 6 + factory/consensus/consensusComponents.go | 2 + 3 files changed, 141 insertions(+), 8 deletions(-) diff --git a/consensus/broadcast/delayedBroadcast.go b/consensus/broadcast/delayedBroadcast.go index 9f67dcbc248..49087024ada 100644 --- a/consensus/broadcast/delayedBroadcast.go +++ b/consensus/broadcast/delayedBroadcast.go @@ -16,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/consensus" "github.com/multiversx/mx-chain-go/consensus/broadcast/shared" "github.com/multiversx/mx-chain-go/consensus/spos" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/sharding" @@ -25,12 +26,16 @@ import ( const prefixHeaderAlarm = "header_" const prefixDelayDataAlarm = "delay_" -const sizeHeadersCache = 1000 // 1000 hashes in cache +const sizeHeadersCache = 1000 +const sizeProcessedMetaHeadersCache = 100 +const maxPendingMetaHeaders = 50 // ArgsDelayedBlockBroadcaster holds the arguments to create a delayed block broadcaster type ArgsDelayedBlockBroadcaster struct { InterceptorsContainer process.InterceptorsContainer HeadersSubscriber consensus.HeadersPoolSubscriber + ProofsPool dataRetriever.ProofsPool + EnableEpochsHandler common.EnableEpochsHandler ShardCoordinator sharding.Coordinator LeaderCacheSize uint32 ValidatorCacheSize uint32 @@ -50,11 +55,19 @@ type headerDataForValidator struct { prevRandSeed []byte } +type pendingHeaderInfo struct { + header data.HeaderHandler + hash []byte + nonce uint64 +} + type delayedBlockBroadcaster struct { alarm timersScheduler interceptorsContainer process.InterceptorsContainer shardCoordinator sharding.Coordinator headersSubscriber consensus.HeadersPoolSubscriber + proofsPool dataRetriever.ProofsPool + enableEpochsHandler common.EnableEpochsHandler valHeaderBroadcastData []*shared.ValidatorHeaderBroadcastData valBroadcastData []*shared.DelayedBroadcastData delayedBroadcastData []*shared.DelayedBroadcastData @@ -67,6 +80,11 @@ type delayedBlockBroadcaster struct { broadcastConsensusMessage func(message *consensus.Message) error cacheHeaders storage.Cacher mutHeadersCache sync.RWMutex + // pendingMetaHeaders stores metachain headers waiting for proof arrival before broadcast. + // mutPendingMetaHeaders and mutDataForBroadcast are never held simultaneously. + pendingMetaHeaders map[string]*pendingHeaderInfo + mutPendingMetaHeaders sync.Mutex + cacheProcessedMetaHeaders storage.Cacher } // NewDelayedBlockBroadcaster create a new instance of a delayed block data broadcaster @@ -83,17 +101,30 @@ func NewDelayedBlockBroadcaster(args *ArgsDelayedBlockBroadcaster) (*delayedBloc if check.IfNil(args.AlarmScheduler) { return nil, spos.ErrNilAlarmScheduler } + if check.IfNil(args.ProofsPool) { + return nil, process.ErrNilProofsPool + } + if check.IfNil(args.EnableEpochsHandler) { + return nil, spos.ErrNilEnableEpochsHandler + } cacheHeaders, err := cache.NewLRUCache(sizeHeadersCache) if err != nil { return nil, err } + cacheProcessedMetaHeaders, err := cache.NewLRUCache(sizeProcessedMetaHeadersCache) + if err != nil { + return nil, err + } + dbb := &delayedBlockBroadcaster{ alarm: args.AlarmScheduler, shardCoordinator: args.ShardCoordinator, interceptorsContainer: args.InterceptorsContainer, headersSubscriber: args.HeadersSubscriber, + proofsPool: args.ProofsPool, + enableEpochsHandler: args.EnableEpochsHandler, valHeaderBroadcastData: make([]*shared.ValidatorHeaderBroadcastData, 0), valBroadcastData: make([]*shared.DelayedBroadcastData, 0), delayedBroadcastData: make([]*shared.DelayedBroadcastData, 0), @@ -102,9 +133,12 @@ func NewDelayedBlockBroadcaster(args *ArgsDelayedBlockBroadcaster) (*delayedBloc mutDataForBroadcast: sync.RWMutex{}, cacheHeaders: cacheHeaders, mutHeadersCache: sync.RWMutex{}, + pendingMetaHeaders: make(map[string]*pendingHeaderInfo), + cacheProcessedMetaHeaders: cacheProcessedMetaHeaders, } dbb.headersSubscriber.RegisterHandler(dbb.headerReceived) + dbb.proofsPool.RegisterHandler(dbb.proofReceived) err = dbb.registerHeaderInterceptorCallback(dbb.interceptedHeader) if err != nil { return nil, err @@ -261,42 +295,133 @@ func (dbb *delayedBlockBroadcaster) Close() { } func (dbb *delayedBlockBroadcaster) headerReceived(headerHandler data.HeaderHandler, headerHash []byte) { + if headerHandler.GetShardID() != core.MetachainShardId { + return + } + + if common.IsProofsFlagEnabledForHeader(dbb.enableEpochsHandler, headerHandler) { + if !dbb.proofsPool.HasProof(headerHandler.GetShardID(), headerHash) { + dbb.addPendingMetaHeader(headerHandler, headerHash) + log.Trace("delayedBlockBroadcaster.headerReceived: proof not yet available, deferring broadcast", + "headerHash", headerHash, + "nonce", headerHandler.GetNonce(), + ) + return + } + } + + dbb.processMetachainHeader(headerHandler, headerHash) +} + +func (dbb *delayedBlockBroadcaster) proofReceived(proof data.HeaderProofHandler) { + if check.IfNil(proof) { + return + } + if proof.GetHeaderShardId() != core.MetachainShardId { + return + } + + headerHash := proof.GetHeaderHash() + hashStr := string(headerHash) + + dbb.mutPendingMetaHeaders.Lock() + pending, found := dbb.pendingMetaHeaders[hashStr] + if found { + delete(dbb.pendingMetaHeaders, hashStr) + } + dbb.evictPendingMetaHeadersUpToNonce(proof.GetHeaderNonce()) + dbb.mutPendingMetaHeaders.Unlock() + + if !found { + return + } + + log.Trace("delayedBlockBroadcaster.proofReceived: proof arrived, triggering deferred broadcast", + "headerHash", headerHash, + "nonce", pending.nonce, + ) + + dbb.processMetachainHeader(pending.header, pending.hash) +} + +func (dbb *delayedBlockBroadcaster) processMetachainHeader(headerHandler data.HeaderHandler, headerHash []byte) { + if alreadyProcessed, _ := dbb.cacheProcessedMetaHeaders.HasOrAdd(headerHash, struct{}{}, 0); alreadyProcessed { + return + } + dbb.mutDataForBroadcast.RLock() defer dbb.mutDataForBroadcast.RUnlock() if len(dbb.delayedBroadcastData) == 0 && len(dbb.valBroadcastData) == 0 { return } - if headerHandler.GetShardID() != core.MetachainShardId { - return - } headerHashes, dataForValidators, err := getShardDataFromMetaChainBlock( headerHandler, dbb.shardCoordinator.SelfId(), ) if err != nil { - log.Error("delayedBlockBroadcaster.headerReceived", "error", err.Error(), + log.Error("delayedBlockBroadcaster.processMetachainHeader", "error", err.Error(), "headerHash", headerHash, ) return } if len(headerHashes) == 0 { - log.Trace("delayedBlockBroadcaster.headerReceived: header received with no shardData for current shard", + log.Trace("delayedBlockBroadcaster.processMetachainHeader: no shardData for current shard", "headerHash", headerHash, ) return } - log.Trace("delayedBlockBroadcaster.headerReceived", "nbHeaderHashes", len(headerHashes)) + log.Trace("delayedBlockBroadcaster.processMetachainHeader", "nbHeaderHashes", len(headerHashes)) for i := range headerHashes { - log.Trace("delayedBlockBroadcaster.headerReceived", "headerHash", headerHashes[i]) + log.Trace("delayedBlockBroadcaster.processMetachainHeader", "headerHash", headerHashes[i]) } go dbb.scheduleValidatorBroadcast(dataForValidators) go dbb.broadcastDataForHeaders(headerHashes) } +func (dbb *delayedBlockBroadcaster) addPendingMetaHeader(header data.HeaderHandler, headerHash []byte) { + dbb.mutPendingMetaHeaders.Lock() + defer dbb.mutPendingMetaHeaders.Unlock() + + if len(dbb.pendingMetaHeaders) >= maxPendingMetaHeaders { + dbb.evictOldestPendingMetaHeader() + } + + dbb.pendingMetaHeaders[string(headerHash)] = &pendingHeaderInfo{ + header: header, + hash: headerHash, + nonce: header.GetNonce(), + } +} + +func (dbb *delayedBlockBroadcaster) evictOldestPendingMetaHeader() { + var oldestKey string + var oldestNonce uint64 + first := true + for key, pending := range dbb.pendingMetaHeaders { + if first || pending.nonce < oldestNonce { + oldestKey = key + oldestNonce = pending.nonce + first = false + } + } + if !first { + delete(dbb.pendingMetaHeaders, oldestKey) + } +} + +// must be called under mutPendingMetaHeaders lock +func (dbb *delayedBlockBroadcaster) evictPendingMetaHeadersUpToNonce(nonce uint64) { + for key, pending := range dbb.pendingMetaHeaders { + if pending.nonce <= nonce { + delete(dbb.pendingMetaHeaders, key) + } + } +} + func (dbb *delayedBlockBroadcaster) broadcastDataForHeaders(headerHashes [][]byte) { dbb.mutDataForBroadcast.RLock() if len(dbb.delayedBroadcastData) == 0 { diff --git a/consensus/spos/sposFactory/sposFactory.go b/consensus/spos/sposFactory/sposFactory.go index 99f0cf682eb..9813a05f422 100644 --- a/consensus/spos/sposFactory/sposFactory.go +++ b/consensus/spos/sposFactory/sposFactory.go @@ -7,10 +7,12 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-crypto-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/consensus" "github.com/multiversx/mx-chain-go/consensus/broadcast" "github.com/multiversx/mx-chain-go/consensus/spos" "github.com/multiversx/mx-chain-go/consensus/spos/bls" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" ) @@ -36,6 +38,8 @@ func GetBroadcastMessenger( interceptorsContainer process.InterceptorsContainer, alarmScheduler core.TimersScheduler, keysHandler consensus.KeysHandler, + proofsPool dataRetriever.ProofsPool, + enableEpochsHandler common.EnableEpochsHandler, ) (consensus.BroadcastMessenger, error) { if check.IfNil(shardCoordinator) { @@ -45,6 +49,8 @@ func GetBroadcastMessenger( dbbArgs := &broadcast.ArgsDelayedBlockBroadcaster{ InterceptorsContainer: interceptorsContainer, HeadersSubscriber: headersSubscriber, + ProofsPool: proofsPool, + EnableEpochsHandler: enableEpochsHandler, ShardCoordinator: shardCoordinator, LeaderCacheSize: maxDelayCacheSize, ValidatorCacheSize: maxDelayCacheSize, diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index 39efa2c4240..131ebdafe07 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -164,6 +164,8 @@ func (ccf *consensusComponentsFactory) Create() (*consensusComponents, error) { ccf.processComponents.InterceptorsContainer(), ccf.coreComponents.AlarmScheduler(), ccf.cryptoComponents.KeysHandler(), + ccf.dataComponents.Datapool().Proofs(), + ccf.coreComponents.EnableEpochsHandler(), ) if err != nil { return nil, err From 722c8a2d3604dc12836a30e9ca11769b83a1da2d Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 12 May 2026 17:44:02 +0300 Subject: [PATCH 035/116] update tests --- consensus/broadcast/delayedBroadcast_test.go | 276 +++++++++++++++++- consensus/broadcast/export.go | 12 + .../broadcast/shardChainMessenger_test.go | 4 + .../spos/sposFactory/sposFactory_test.go | 10 + integrationTests/testFullNode.go | 15 +- integrationTests/testProcessorNode.go | 4 + .../components/testOnlyProcessingNode.go | 2 + 7 files changed, 317 insertions(+), 6 deletions(-) diff --git a/consensus/broadcast/delayedBroadcast_test.go b/consensus/broadcast/delayedBroadcast_test.go index 961bc0efcc9..d302aa2d593 100644 --- a/consensus/broadcast/delayedBroadcast_test.go +++ b/consensus/broadcast/delayedBroadcast_test.go @@ -26,6 +26,8 @@ import ( "github.com/multiversx/mx-chain-go/consensus/spos" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/testscommon" + dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" + "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/pool" ) @@ -137,6 +139,8 @@ func createDefaultDelayedBroadcasterArgs() *broadcast.ArgsDelayedBlockBroadcaste ShardCoordinator: &mock.ShardCoordinatorMock{}, InterceptorsContainer: interceptorsContainer, HeadersSubscriber: headersSubscriber, + ProofsPool: &dataRetrieverMock.ProofsPoolMock{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, LeaderCacheSize: 2, ValidatorCacheSize: 2, AlarmScheduler: alarm.NewAlarmScheduler(), @@ -185,6 +189,26 @@ func TestNewDelayedBlockBroadcaster_NilAlarmSchedulerShouldErr(t *testing.T) { require.Nil(t, dbb) } +func TestNewDelayedBlockBroadcaster_NilProofsPoolShouldErr(t *testing.T) { + t.Parallel() + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + delayBroadcasterArgs.ProofsPool = nil + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Equal(t, process.ErrNilProofsPool, err) + require.Nil(t, dbb) +} + +func TestNewDelayedBlockBroadcaster_NilEnableEpochsHandlerShouldErr(t *testing.T) { + t.Parallel() + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + delayBroadcasterArgs.EnableEpochsHandler = nil + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Equal(t, spos.ErrNilEnableEpochsHandler, err) + require.Nil(t, dbb) +} + func TestNewDelayedBlockBroadcasterOK(t *testing.T) { t.Parallel() @@ -387,7 +411,7 @@ func TestDelayedBlockBroadcaster_HeaderReceivedWithoutSignaturesForShardShouldNo time.Sleep(sleepTime) logOutputStr := observer.getBufferStr() - expectedLogMsg := "delayedBlockBroadcaster.headerReceived: header received with no shardData for current shard" + expectedLogMsg := "delayedBlockBroadcaster.processMetachainHeader: no shardData for current shard" require.Contains(t, logOutputStr, expectedLogMsg) require.Contains(t, logOutputStr, fmt.Sprintf("headerHash = %s", hex.EncodeToString(headerHash))) @@ -1889,3 +1913,253 @@ func TestDelayedBlockBroadcaster_Close(t *testing.T) { vbd = dbb.GetValidatorBroadcastData() require.Equal(t, 1, len(vbd)) } + +func TestDelayedBlockBroadcaster_HeaderReceivedWithProofsEnabled_DefersUntilProof(t *testing.T) { + t.Parallel() + + mbBroadcastCalled := atomic.Flag{} + txBroadcastCalled := atomic.Flag{} + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + delayBroadcasterArgs.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + + hasProof := false + delayBroadcasterArgs.ProofsPool = &dataRetrieverMock.ProofsPoolMock{ + HasProofCalled: func(shardID uint32, headerHash []byte) bool { + return hasProof + }, + } + + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Nil(t, err) + + err = dbb.SetBroadcastHandlers( + func(mbData map[uint32][]byte, pk []byte) error { + mbBroadcastCalled.SetValue(true) + return nil + }, + func(txData map[string][][]byte, pk []byte) error { + txBroadcastCalled.SetValue(true) + return nil + }, + func(header data.HeaderHandler, pk []byte) error { return nil }, + func(message *consensus.Message) error { return nil }, + ) + require.Nil(t, err) + + headerHash, _, miniblocksData, transactionsData := createDelayData("1") + delayedData := broadcast.CreateDelayBroadcastDataForLeader(headerHash, miniblocksData, transactionsData) + err = dbb.SetLeaderData(delayedData) + require.Nil(t, err) + + metaBlock := createMetaBlock() + metaBlock.ShardInfo[0].HeaderHash = headerHash + metaBlock.Epoch = 1 + metaBlock.Nonce = 10 + metaHash := []byte("meta hash") + + dbb.HeaderReceived(metaBlock, metaHash) + + sleepTime := common.ExtraDelayForBroadcastBlockInfo + + common.ExtraDelayBetweenBroadcastMbsAndTxs + + 100*time.Millisecond + time.Sleep(sleepTime) + + assert.False(t, mbBroadcastCalled.IsSet(), "should not broadcast without proof") + assert.False(t, txBroadcastCalled.IsSet(), "should not broadcast without proof") + assert.Equal(t, 1, dbb.GetPendingMetaHeadersCount(), "header should be pending") + + hasProof = true + proof := &block.HeaderProof{ + HeaderHash: metaHash, + HeaderShardId: core.MetachainShardId, + HeaderNonce: 10, + HeaderEpoch: 1, + } + dbb.ProofReceived(proof) + + time.Sleep(sleepTime) + + assert.True(t, mbBroadcastCalled.IsSet(), "should broadcast after proof arrives") + assert.True(t, txBroadcastCalled.IsSet(), "should broadcast after proof arrives") + assert.Equal(t, 0, dbb.GetPendingMetaHeadersCount(), "pending should be cleared") +} + +func TestDelayedBlockBroadcaster_ProofReceivedEvictsOlderNonces(t *testing.T) { + t.Parallel() + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + delayBroadcasterArgs.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + delayBroadcasterArgs.ProofsPool = &dataRetrieverMock.ProofsPoolMock{ + HasProofCalled: func(shardID uint32, headerHash []byte) bool { + return false + }, + } + + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Nil(t, err) + + err = dbb.SetBroadcastHandlers( + func(mbData map[uint32][]byte, pk []byte) error { return nil }, + func(txData map[string][][]byte, pk []byte) error { return nil }, + func(header data.HeaderHandler, pk []byte) error { return nil }, + func(message *consensus.Message) error { return nil }, + ) + require.Nil(t, err) + + for i := 0; i < 3; i++ { + headerHash, _, miniblocksData, transactionsData := createDelayData(strconv.Itoa(i)) + delayedData := broadcast.CreateDelayBroadcastDataForLeader(headerHash, miniblocksData, transactionsData) + err = dbb.SetLeaderData(delayedData) + require.Nil(t, err) + + metaBlock := createMetaBlock() + metaBlock.ShardInfo[0].HeaderHash = headerHash + metaBlock.Epoch = 1 + metaBlock.Nonce = uint64(10 + i) + + dbb.HeaderReceived(metaBlock, []byte(fmt.Sprintf("meta hash %d", i))) + } + + assert.Equal(t, 3, dbb.GetPendingMetaHeadersCount()) + + proof := &block.HeaderProof{ + HeaderHash: []byte("unknown hash"), + HeaderShardId: core.MetachainShardId, + HeaderNonce: 11, + HeaderEpoch: 1, + } + dbb.ProofReceived(proof) + + assert.Equal(t, 1, dbb.GetPendingMetaHeadersCount(), "only nonce 12 should remain") +} + +func TestDelayedBlockBroadcaster_HeaderReceivedWithProofsEnabled_ProofAlreadyAvailable(t *testing.T) { + t.Parallel() + + mbBroadcastCalled := atomic.Flag{} + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + delayBroadcasterArgs.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + delayBroadcasterArgs.ProofsPool = &dataRetrieverMock.ProofsPoolMock{ + HasProofCalled: func(shardID uint32, headerHash []byte) bool { + return true + }, + } + + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Nil(t, err) + + err = dbb.SetBroadcastHandlers( + func(mbData map[uint32][]byte, pk []byte) error { + mbBroadcastCalled.SetValue(true) + return nil + }, + func(txData map[string][][]byte, pk []byte) error { return nil }, + func(header data.HeaderHandler, pk []byte) error { return nil }, + func(message *consensus.Message) error { return nil }, + ) + require.Nil(t, err) + + headerHash, _, miniblocksData, transactionsData := createDelayData("1") + delayedData := broadcast.CreateDelayBroadcastDataForLeader(headerHash, miniblocksData, transactionsData) + err = dbb.SetLeaderData(delayedData) + require.Nil(t, err) + + metaBlock := createMetaBlock() + metaBlock.ShardInfo[0].HeaderHash = headerHash + metaBlock.Epoch = 1 + metaBlock.Nonce = 10 + + dbb.HeaderReceived(metaBlock, []byte("meta hash")) + + sleepTime := common.ExtraDelayForBroadcastBlockInfo + + common.ExtraDelayBetweenBroadcastMbsAndTxs + + 100*time.Millisecond + time.Sleep(sleepTime) + + assert.True(t, mbBroadcastCalled.IsSet(), "should broadcast immediately when proof is already available") + assert.Equal(t, 0, dbb.GetPendingMetaHeadersCount()) +} + +func TestDelayedBlockBroadcaster_DuplicateProcessingPrevented(t *testing.T) { + t.Parallel() + + broadcastCount := atomic.Counter{} + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + delayBroadcasterArgs.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + delayBroadcasterArgs.ProofsPool = &dataRetrieverMock.ProofsPoolMock{ + HasProofCalled: func(shardID uint32, headerHash []byte) bool { + return true + }, + } + + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Nil(t, err) + + err = dbb.SetBroadcastHandlers( + func(mbData map[uint32][]byte, pk []byte) error { + broadcastCount.Increment() + return nil + }, + func(txData map[string][][]byte, pk []byte) error { return nil }, + func(header data.HeaderHandler, pk []byte) error { return nil }, + func(message *consensus.Message) error { return nil }, + ) + require.Nil(t, err) + + headerHash, _, miniblocksData, transactionsData := createDelayData("1") + delayedData := broadcast.CreateDelayBroadcastDataForLeader(headerHash, miniblocksData, transactionsData) + err = dbb.SetLeaderData(delayedData) + require.Nil(t, err) + + metaBlock := createMetaBlock() + metaBlock.ShardInfo[0].HeaderHash = headerHash + metaBlock.Epoch = 1 + metaBlock.Nonce = 10 + metaHash := []byte("meta hash") + + dbb.HeaderReceived(metaBlock, metaHash) + dbb.HeaderReceived(metaBlock, metaHash) + + sleepTime := common.ExtraDelayForBroadcastBlockInfo + + common.ExtraDelayBetweenBroadcastMbsAndTxs + + 100*time.Millisecond + time.Sleep(sleepTime) + + assert.Equal(t, int64(1), broadcastCount.Get(), "should broadcast only once despite two HeaderReceived calls") +} + +func TestDelayedBlockBroadcaster_ProofReceivedNonMetaShouldBeIgnored(t *testing.T) { + t.Parallel() + + delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() + dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) + require.Nil(t, err) + + proof := &block.HeaderProof{ + HeaderHash: []byte("some hash"), + HeaderShardId: 0, + HeaderNonce: 10, + } + dbb.ProofReceived(proof) + + assert.Equal(t, 0, dbb.GetPendingMetaHeadersCount()) +} diff --git a/consensus/broadcast/export.go b/consensus/broadcast/export.go index 5351003e38f..fbde6330f1a 100644 --- a/consensus/broadcast/export.go +++ b/consensus/broadcast/export.go @@ -81,6 +81,18 @@ func (dbb *delayedBlockBroadcaster) HeaderReceived(headerHandler data.HeaderHand dbb.headerReceived(headerHandler, hash) } +// ProofReceived is the callback for when a proof is received +func (dbb *delayedBlockBroadcaster) ProofReceived(proof data.HeaderProofHandler) { + dbb.proofReceived(proof) +} + +// GetPendingMetaHeadersCount returns the number of pending meta headers +func (dbb *delayedBlockBroadcaster) GetPendingMetaHeadersCount() int { + dbb.mutPendingMetaHeaders.Lock() + defer dbb.mutPendingMetaHeaders.Unlock() + return len(dbb.pendingMetaHeaders) +} + // GetValidatorBroadcastData returns the set validator delayed broadcast data func (dbb *delayedBlockBroadcaster) GetValidatorBroadcastData() []*shared.DelayedBroadcastData { dbb.mutDataForBroadcast.RLock() diff --git a/consensus/broadcast/shardChainMessenger_test.go b/consensus/broadcast/shardChainMessenger_test.go index 7846ba12b0d..c349f7fd46a 100644 --- a/consensus/broadcast/shardChainMessenger_test.go +++ b/consensus/broadcast/shardChainMessenger_test.go @@ -26,6 +26,8 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/testscommon" + dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" + "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" ) @@ -572,6 +574,8 @@ func TestShardChainMessenger_BroadcastBlockDataLeaderShouldTriggerWaitingDelayed argsDelayedBroadcaster := broadcast.ArgsDelayedBlockBroadcaster{ InterceptorsContainer: args.InterceptorsContainer, HeadersSubscriber: args.HeadersSubscriber, + ProofsPool: &dataRetrieverMock.ProofsPoolMock{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ShardCoordinator: args.ShardCoordinator, LeaderCacheSize: args.MaxDelayCacheSize, ValidatorCacheSize: args.MaxDelayCacheSize, diff --git a/consensus/spos/sposFactory/sposFactory_test.go b/consensus/spos/sposFactory/sposFactory_test.go index 1f122884530..8aac724a088 100644 --- a/consensus/spos/sposFactory/sposFactory_test.go +++ b/consensus/spos/sposFactory/sposFactory_test.go @@ -12,6 +12,8 @@ import ( "github.com/multiversx/mx-chain-go/consensus/spos" "github.com/multiversx/mx-chain-go/consensus/spos/sposFactory" "github.com/multiversx/mx-chain-go/testscommon" + dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" + "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" "github.com/multiversx/mx-chain-go/testscommon/pool" @@ -60,6 +62,8 @@ func TestGetBroadcastMessenger_ShardShouldWork(t *testing.T) { interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, + &dataRetrieverMock.ProofsPoolMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, err) @@ -91,6 +95,8 @@ func TestGetBroadcastMessenger_MetachainShouldWork(t *testing.T) { interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, + &dataRetrieverMock.ProofsPoolMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, err) @@ -114,6 +120,8 @@ func TestGetBroadcastMessenger_NilShardCoordinatorShouldErr(t *testing.T) { interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, + &dataRetrieverMock.ProofsPoolMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, bm) @@ -141,6 +149,8 @@ func TestGetBroadcastMessenger_InvalidShardIdShouldErr(t *testing.T) { interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, + &dataRetrieverMock.ProofsPoolMock{}, + &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, bm) diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 4c122860f52..559742861ef 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -14,9 +14,10 @@ import ( crypto "github.com/multiversx/mx-chain-crypto-go" mclMultiSig "github.com/multiversx/mx-chain-crypto-go/signing/mcl/multisig" "github.com/multiversx/mx-chain-crypto-go/signing/multisig" - "github.com/multiversx/mx-chain-go/state/disabled" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" + "github.com/multiversx/mx-chain-go/state/disabled" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/enablers" "github.com/multiversx/mx-chain-go/common/forking" @@ -343,6 +344,8 @@ func (tpn *TestFullNode) initTestNodeWithArgs(args ArgTestProcessorNode, fullArg tpn.NodeKeys.MainKey.Sk, tpn.MainMessenger.ID(), ), + tpn.DataPool.Proofs(), + tpn.EnableEpochsHandler, ) if args.WithSync { @@ -600,7 +603,7 @@ func (tpn *TestFullNode) initNode( } func (tfn *TestFullNode) createForkDetector( - startTime int64, + _ int64, roundHandler consensus.RoundHandler, ) process.ForkDetector { var err error @@ -631,7 +634,7 @@ func (tfn *TestFullNode) createForkDetector( return forkDetector } -func (tfn *TestFullNode) createEpochStartTrigger(startTime int64) TestEpochStartTrigger { +func (tfn *TestFullNode) createEpochStartTrigger(_ int64) TestEpochStartTrigger { var epochTrigger TestEpochStartTrigger if tfn.ShardCoordinator.SelfId() == core.MetachainShardId { argsNewMetaEpochStart := &metachain.ArgsNewMetaEpochStartTrigger{ @@ -753,6 +756,7 @@ func (tcn *TestFullNode) initInterceptors( interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) if err != nil { fmt.Println(err.Error()) + return } tcn.MainInterceptorsContainer, _, err = interceptorContainerFactory.Create() @@ -788,6 +792,7 @@ func (tcn *TestFullNode) initInterceptors( interceptorContainerFactory, err := interceptorscontainer.NewShardInterceptorsContainerFactory(interceptorContainerFactoryArgs) if err != nil { fmt.Println(err.Error()) + return } tcn.MainInterceptorsContainer, _, err = interceptorContainerFactory.Create() @@ -800,7 +805,7 @@ func (tcn *TestFullNode) initInterceptors( func (tpn *TestFullNode) initBlockProcessor( coreComponents *mock.CoreComponentsStub, dataComponents *mock.DataComponentsStub, - args ArgsTestFullNode, + _ ArgsTestFullNode, roundHandler consensus.RoundHandler, ) { var err error @@ -1041,7 +1046,7 @@ func (tpn *TestFullNode) initBlockProcessor( func (tpn *TestFullNode) initBlockProcessorWithSync( coreComponents *mock.CoreComponentsStub, dataComponents *mock.DataComponentsStub, - roundHandler consensus.RoundHandler, + _ consensus.RoundHandler, ) { var err error diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 3344722fb65..fcb8946553e 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -867,6 +867,8 @@ func (tpn *TestProcessorNode) initTestNodeWithArgs(args ArgTestProcessorNode) { tpn.NodeKeys.MainKey.Sk, tpn.MainMessenger.ID(), ), + tpn.DataPool.Proofs(), + tpn.EnableEpochsHandler, ) if args.WithSync { @@ -1095,6 +1097,8 @@ func (tpn *TestProcessorNode) InitializeProcessors(gasMap map[string]map[string] tpn.NodeKeys.MainKey.Sk, tpn.MainMessenger.ID(), ), + tpn.DataPool.Proofs(), + tpn.EnableEpochsHandler, ) tpn.setGenesisBlock() tpn.initNode() diff --git a/node/chainSimulator/components/testOnlyProcessingNode.go b/node/chainSimulator/components/testOnlyProcessingNode.go index 8e6148f40f5..059f1a2733b 100644 --- a/node/chainSimulator/components/testOnlyProcessingNode.go +++ b/node/chainSimulator/components/testOnlyProcessingNode.go @@ -338,6 +338,8 @@ func (node *testOnlyProcessingNode) createBroadcastMessenger() error { node.ProcessComponentsHolder.InterceptorsContainer(), node.CoreComponentsHolder.AlarmScheduler(), node.CryptoComponentsHolder.KeysHandler(), + node.DataComponentsHolder.Datapool().Proofs(), + node.CoreComponentsHolder.EnableEpochsHandler(), ) if err != nil { return err From 0ded82a56ab011cabaf1a05b4ba53e3f60283f10 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 13 May 2026 11:36:55 +0300 Subject: [PATCH 036/116] update tests --- common/common_test.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/common/common_test.go b/common/common_test.go index 22dd024821e..a0f8194611f 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -9,12 +9,13 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/chainParameters" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" - "github.com/stretchr/testify/require" ) var testFlag = core.EnableEpochFlag("test flag") @@ -105,6 +106,29 @@ func TestIsConsensusBitmapValid(t *testing.T) { require.Equal(t, common.ErrNotEnoughSignatures, err) }) + t.Run("padding bits set should return error", func(t *testing.T) { + t.Parallel() + + // consensus size is 10, so bitmap should have 2 bytes + bitmap := make([]byte, len(pubKeys)/8+1) + bitmap[0] = 0xFF + bitmap[1] = 0x07 + + err := common.IsConsensusBitmapValid(log, pubKeys, bitmap, false) + require.Equal(t, common.ErrPaddingBitsSet, err) + }) + + t.Run("padding bits not set should return nil", func(t *testing.T) { + t.Parallel() + + bitmap := make([]byte, len(pubKeys)/8+1) + bitmap[0] = 0xFF + bitmap[1] = 0x03 + + err := common.IsConsensusBitmapValid(log, pubKeys, bitmap, false) + require.Nil(t, err) + }) + t.Run("should work", func(t *testing.T) { t.Parallel() From 1c5e3fbaf9a8680eca903801cc26a28f81bd266c Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 13 May 2026 13:17:01 +0300 Subject: [PATCH 037/116] move intercepted messages checks before Verify call --- .../shardRequestersContainerFactory.go | 5 ++ .../disabled/resolversContainerFactory.go | 5 ++ .../shardResolversContainerFactory.go | 5 ++ .../metaRequestersContainerFactory.go | 5 ++ .../shardRequestersContainerFactory.go | 5 ++ dataRetriever/interface.go | 2 + .../epochStartInterceptorsContainerFactory.go | 7 ++- epochStart/bootstrap/fromLocalStorage.go | 10 ++-- epochStart/bootstrap/process.go | 34 +++++++++-- .../interceptedMetaBlockHeader.go | 4 ++ .../shardInterceptorsContainerFactory.go | 5 ++ process/interceptors/multiDataInterceptor.go | 58 ++++++++++--------- process/interceptors/singleDataInterceptor.go | 33 ++++++----- .../singleDataInterceptor_test.go | 2 +- process/interface.go | 1 + update/factory/fullSyncInterceptors.go | 5 ++ 16 files changed, 131 insertions(+), 55 deletions(-) diff --git a/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go b/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go index 014cef057c3..31607ca3a0b 100644 --- a/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go +++ b/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go @@ -93,6 +93,11 @@ func (srcf *shardRequestersContainerFactory) Create() (dataRetriever.RequestersC return srcf.container, nil } +// AddShardTrieNodeRequesters returns nil +func (srcf *shardRequestersContainerFactory) AddShardTrieNodeRequesters(_ dataRetriever.RequestersContainer) error { + return nil +} + func (srcf *shardRequestersContainerFactory) generateHeaderRequesters() error { shardC := srcf.shardCoordinator diff --git a/dataRetriever/factory/resolverscontainer/disabled/resolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/disabled/resolversContainerFactory.go index e96ebc8c5f0..7d7b7b3abd6 100644 --- a/dataRetriever/factory/resolverscontainer/disabled/resolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/disabled/resolversContainerFactory.go @@ -18,6 +18,11 @@ func (rcf *resolversContainerFactory) Create() (dataRetriever.ResolversContainer return disabled.NewDisabledResolversContainer(), nil } +// AddShardTrieNodeResolvers returns nil as it is disabled +func (rcf *resolversContainerFactory) AddShardTrieNodeResolvers(_ dataRetriever.ResolversContainer) error { + return nil +} + // IsInterfaceNil returns true if there is no value under the interface func (rcf *resolversContainerFactory) IsInterfaceNil() bool { return rcf == nil diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go index 3c1f374e4a8..d8179d0e63f 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go @@ -137,6 +137,11 @@ func (srcf *shardResolversContainerFactory) Create() (dataRetriever.ResolversCon return srcf.container, nil } +// AddShardTrieNodeResolvers returns nil +func (srcf *shardResolversContainerFactory) AddShardTrieNodeResolvers(_ dataRetriever.ResolversContainer) error { + return nil +} + // ------- Hdr resolver func (srcf *shardResolversContainerFactory) generateHeaderResolvers() error { diff --git a/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory.go b/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory.go index e430ff170dc..5d3c947d930 100644 --- a/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory.go +++ b/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory.go @@ -83,6 +83,11 @@ func (mrcf *metaRequestersContainerFactory) Create() (dataRetriever.RequestersCo return mrcf.container, nil } +// AddShardTrieNodeRequesters returns nil +func (mrcf *metaRequestersContainerFactory) AddShardTrieNodeRequesters(_ dataRetriever.RequestersContainer) error { + return nil +} + func (mrcf *metaRequestersContainerFactory) generateShardHeaderRequesters() error { shardC := mrcf.shardCoordinator noOfShards := shardC.NumberOfShards() diff --git a/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory.go b/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory.go index 2380a6380cd..bfbd20c9497 100644 --- a/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory.go +++ b/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory.go @@ -83,6 +83,11 @@ func (srcf *shardRequestersContainerFactory) Create() (dataRetriever.RequestersC return srcf.container, nil } +// AddShardTrieNodeRequesters returns nil +func (srcf *shardRequestersContainerFactory) AddShardTrieNodeRequesters(_ dataRetriever.RequestersContainer) error { + return nil +} + func (srcf *shardRequestersContainerFactory) generateHeaderRequesters() error { shardC := srcf.shardCoordinator diff --git a/dataRetriever/interface.go b/dataRetriever/interface.go index f9d68fa9f92..b95be5d7503 100644 --- a/dataRetriever/interface.go +++ b/dataRetriever/interface.go @@ -85,6 +85,7 @@ type RequestersFinder interface { // ResolversContainerFactory defines the functionality to create a resolvers container type ResolversContainerFactory interface { Create() (ResolversContainer, error) + AddShardTrieNodeResolvers(container ResolversContainer) error IsInterfaceNil() bool } @@ -117,6 +118,7 @@ type RequestersContainer interface { // RequestersContainerFactory defines the functionality to create a requesters container type RequestersContainerFactory interface { Create() (RequestersContainer, error) + AddShardTrieNodeRequesters(container RequestersContainer) error IsInterfaceNil() bool } diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index 8700b1daa24..bb5054005be 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -113,7 +113,12 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } - interceptorsContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) + var interceptorsContainerFactory process.InterceptorsContainerFactory + if args.ShardCoordinator.SelfId() == core.MetachainShardId { + interceptorsContainerFactory, err = interceptorscontainer.NewMetaInterceptorsContainerFactory(containerFactoryArgs) + } else { + interceptorsContainerFactory, err = interceptorscontainer.NewShardInterceptorsContainerFactory(containerFactoryArgs) + } if err != nil { return nil, nil, err } diff --git a/epochStart/bootstrap/fromLocalStorage.go b/epochStart/bootstrap/fromLocalStorage.go index 0572d3b376e..e4dd4e1c3a9 100644 --- a/epochStart/bootstrap/fromLocalStorage.go +++ b/epochStart/bootstrap/fromLocalStorage.go @@ -127,6 +127,11 @@ func (e *epochStartBootstrap) prepareEpochFromStorage() (Parameters, error) { log.Debug("prepareEpochFromStorage for shuffled out", "initial shard id", e.baseData.shardId, "new shard id", newShardId) e.baseData.shardId = newShardId + e.shardCoordinator, err = sharding.NewMultiShardCoordinator(e.baseData.numberOfShards, e.baseData.shardId) + if err != nil { + return Parameters{}, err + } + err = e.createRequestHandler() if err != nil { return Parameters{}, err @@ -161,11 +166,6 @@ func (e *epochStartBootstrap) prepareEpochFromStorage() (Parameters, error) { } e.prevEpochStartMeta = prevEpochStartMeta - e.shardCoordinator, err = sharding.NewMultiShardCoordinator(e.baseData.numberOfShards, e.baseData.shardId) - if err != nil { - return Parameters{}, err - } - consensusTopic := common.ConsensusTopic + e.shardCoordinator.CommunicationIdentifier(e.shardCoordinator.SelfId()) err = e.mainMessenger.CreateTopic(consensusTopic, true) if err != nil { diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index a89a36dc6ec..4644742434f 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -358,8 +358,14 @@ func (e *epochStartBootstrap) Bootstrap() (Parameters, error) { defer e.cleanupOnBootstrapFinish() - var err error - e.shardCoordinator, err = sharding.NewMultiShardCoordinator(e.genesisShardCoordinator.NumberOfShards(), core.MetachainShardId) + newShardId, _, err := e.getShardIDForLatestEpoch() + if err != nil { + // fallback to meta if nothing was loaded from the last epoch + newShardId = core.MetachainShardId + } + log.Debug("epochStartBootstrap.Bootstrap", "newShardId", newShardId, "from last epoch", err == nil) + + e.shardCoordinator, err = sharding.NewMultiShardCoordinator(e.genesisShardCoordinator.NumberOfShards(), newShardId) if err != nil { return Parameters{}, err } @@ -674,7 +680,13 @@ func (e *epochStartBootstrap) syncHeadersFrom(meta data.MetaHeaderHandler) (map[ if err != nil { return nil, err } + + isCurrentShardMeta := e.shardCoordinator.SelfId() == core.MetachainShardId for _, epochStartData := range meta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers() { + if !isCurrentShardMeta && epochStartData.GetShardID() != e.shardCoordinator.SelfId() { + continue + } + hashesToRequest = append(hashesToRequest, epochStartData.GetHeaderHash()) shardIds = append(shardIds, epochStartData.GetShardID()) @@ -1480,7 +1492,12 @@ func (e *epochStartBootstrap) createResolversContainer() error { FullArchivePreferredPeersHolder: disabled.NewPreferredPeersHolder(), PayloadValidator: payloadValidator, } - resolverFactory, err := resolverscontainer.NewMetaResolversContainerFactory(resolversContainerArgs) + var resolverFactory dataRetriever.ResolversContainerFactory + if e.shardCoordinator.SelfId() == core.MetachainShardId { + resolverFactory, err = resolverscontainer.NewMetaResolversContainerFactory(resolversContainerArgs) + } else { + resolverFactory, err = resolverscontainer.NewShardResolversContainerFactory(resolversContainerArgs) + } if err != nil { return err } @@ -1509,7 +1526,14 @@ func (e *epochStartBootstrap) createRequestHandler() error { SizeCheckDelta: 0, EnableEpochsHandler: e.enableEpochsHandler, } - requestersFactory, err := requesterscontainer.NewMetaRequestersContainerFactory(requestersContainerArgs) + + var requestersFactory dataRetriever.RequestersContainerFactory + var err error + if e.shardCoordinator.SelfId() == core.MetachainShardId { + requestersFactory, err = requesterscontainer.NewMetaRequestersContainerFactory(requestersContainerArgs) + } else { + requestersFactory, err = requesterscontainer.NewShardRequestersContainerFactory(requestersContainerArgs) + } if err != nil { return err } @@ -1535,7 +1559,7 @@ func (e *epochStartBootstrap) createRequestHandler() error { requestedItemsHandler, e.whiteListHandler, maxToRequest, - core.MetachainShardId, + e.shardCoordinator.SelfId(), timeBetweenRequests, time.Duration(e.generalConfig.Requesters.RequestProofByNonceDelayMs)*time.Millisecond, ) diff --git a/process/block/interceptedBlocks/interceptedMetaBlockHeader.go b/process/block/interceptedBlocks/interceptedMetaBlockHeader.go index 0f85553807d..37343061166 100644 --- a/process/block/interceptedBlocks/interceptedMetaBlockHeader.go +++ b/process/block/interceptedBlocks/interceptedMetaBlockHeader.go @@ -134,6 +134,10 @@ func (imh *InterceptedMetaHeader) isMetaHeaderEpochOutOfRange() bool { return false } + if imh.epochStartTrigger.Epoch() == 0 { + return false + } + if imh.hdr.GetEpoch() > imh.epochStartTrigger.Epoch()+1 { return true } diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index d144113d30f..6ce24a4a32f 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -210,6 +210,11 @@ func (sicf *shardInterceptorsContainerFactory) Create() (process.InterceptorsCon return sicf.mainContainer, sicf.fullArchiveContainer, nil } +// AddShardTrieNodeInterceptors returns nil +func (sicf *shardInterceptorsContainerFactory) AddShardTrieNodeInterceptors(_ process.InterceptorsContainer) error { + return nil +} + func (sicf *shardInterceptorsContainerFactory) generateTrieNodesInterceptors() error { shardC := sicf.shardCoordinator diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 76b33046b03..643457286d8 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -165,38 +165,13 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, for index, dataBuff := range multiDataBuff { var interceptedData process.InterceptedData - interceptedData, err = mdi.interceptedData(dataBuff, message.Peer(), fromConnectedPeer) + interceptedData, err = mdi.interceptedData(dataBuff, message, fromConnectedPeer, errOriginator) listInterceptedData[index] = interceptedData if err != nil { mdi.throttler.EndProcessing() return nil, err } - - isWhiteListed := mdi.whiteListRequest.IsWhiteListed(interceptedData) - if !isWhiteListed && errOriginator != nil { - mdi.throttler.EndProcessing() - log.Trace("got message from peer on topic only for validators", "originator", - p2p.PeerIdToShortString(message.Peer()), - "topic", mdi.topic, - "err", errOriginator) - return nil, errOriginator - } - - isForCurrentShard := interceptedData.IsForCurrentShard() - shouldProcess := isForCurrentShard || isWhiteListed - if !shouldProcess { - log.Trace("intercepted data should not be processed", - "pid", p2p.MessageOriginatorPid(message), - "seq no", p2p.MessageOriginatorSeq(message), - "topic", message.Topic(), - "hash", interceptedData.Hash(), - "is for this shard", isForCurrentShard, - "is white listed", isWhiteListed, - ) - mdi.throttler.EndProcessing() - return nil, process.ErrInterceptedDataNotForCurrentShard - } } go func() { @@ -231,7 +206,13 @@ func (mdi *MultiDataInterceptor) createInterceptedMultiDataMsgID(interceptedMult return mdi.hasher.Compute(string(data)) } -func (mdi *MultiDataInterceptor) interceptedData(dataBuff []byte, originator core.PeerID, fromConnectedPeer core.PeerID) (process.InterceptedData, error) { +func (mdi *MultiDataInterceptor) interceptedData( + dataBuff []byte, + message p2p.MessageP2P, + fromConnectedPeer core.PeerID, + errOriginator error, +) (process.InterceptedData, error) { + originator := message.Peer() interceptedData, err := mdi.factory.Create(dataBuff, originator) if err != nil { // this situation is so severe that we need to black list de peers @@ -244,6 +225,29 @@ func (mdi *MultiDataInterceptor) interceptedData(dataBuff []byte, originator cor mdi.receivedDebugInterceptedData(interceptedData) + isWhiteListed := mdi.whiteListRequest.IsWhiteListed(interceptedData) + if !isWhiteListed && errOriginator != nil { + log.Trace("got message from peer on topic only for validators", "originator", + p2p.PeerIdToShortString(originator), + "topic", mdi.topic, + "err", errOriginator) + return nil, errOriginator + } + + isForCurrentShard := interceptedData.IsForCurrentShard() + shouldProcess := isForCurrentShard || isWhiteListed + if !shouldProcess { + log.Trace("intercepted data should not be processed", + "pid", p2p.MessageOriginatorPid(message), + "seq no", p2p.MessageOriginatorSeq(message), + "topic", message.Topic(), + "hash", interceptedData.Hash(), + "is for this shard", isForCurrentShard, + "is white listed", isWhiteListed, + ) + return nil, process.ErrInterceptedDataNotForCurrentShard + } + err = mdi.interceptedDataVerifier.Verify(interceptedData) if err != nil { mdi.processDebugInterceptedData(interceptedData, err) diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index da15d00170e..8d0f1075fa0 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -101,21 +101,6 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, } sdi.receivedDebugInterceptedData(interceptedData) - err = sdi.interceptedDataVerifier.Verify(interceptedData) - if err != nil { - sdi.throttler.EndProcessing() - sdi.processDebugInterceptedData(interceptedData, err) - - isWrongVersion := errors.Is(err, process.ErrInvalidTransactionVersion) || errors.Is(err, process.ErrInvalidChainID) - if isWrongVersion { - // this situation is so severe that we need to black list de peers - reason := "wrong version of received intercepted data, topic " + sdi.topic + ", error " + err.Error() - sdi.antifloodHandler.BlacklistPeer(message.Peer(), reason, common.InvalidMessageBlacklistDuration) - sdi.antifloodHandler.BlacklistPeer(fromConnectedPeer, reason, common.InvalidMessageBlacklistDuration) - } - - return nil, err - } errOriginator := sdi.antifloodHandler.IsOriginatorEligibleForTopic(message.Peer(), sdi.topic) isWhiteListed := sdi.whiteListRequest.IsWhiteListed(interceptedData) @@ -141,7 +126,23 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, "is white listed", isWhiteListed, ) - return messageID, nil + return messageID, process.ErrInterceptedDataNotForCurrentShard + } + + err = sdi.interceptedDataVerifier.Verify(interceptedData) + if err != nil { + sdi.throttler.EndProcessing() + sdi.processDebugInterceptedData(interceptedData, err) + + isWrongVersion := errors.Is(err, process.ErrInvalidTransactionVersion) || errors.Is(err, process.ErrInvalidChainID) + if isWrongVersion { + // this situation is so severe that we need to black list de peers + reason := "wrong version of received intercepted data, topic " + sdi.topic + ", error " + err.Error() + sdi.antifloodHandler.BlacklistPeer(message.Peer(), reason, common.InvalidMessageBlacklistDuration) + sdi.antifloodHandler.BlacklistPeer(fromConnectedPeer, reason, common.InvalidMessageBlacklistDuration) + } + + return nil, err } go func() { diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index 84aa285ff6c..d64ee682359 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -238,7 +238,7 @@ func TestSingleDataInterceptor_ProcessReceivedMessageIsNotValidShouldNotCallProc func TestSingleDataInterceptor_ProcessReceivedMessageIsNotForCurrentShardShouldNotCallProcess(t *testing.T) { t.Parallel() - testProcessReceiveMessage(t, false, nil, 0) + testProcessReceiveMessage(t, false, process.ErrInterceptedDataNotForCurrentShard, 0) } func TestSingleDataInterceptor_ProcessReceivedMessageShouldWork(t *testing.T) { diff --git a/process/interface.go b/process/interface.go index 99bafaa1354..d1bb189de17 100644 --- a/process/interface.go +++ b/process/interface.go @@ -407,6 +407,7 @@ type InterceptorsContainer interface { // InterceptorsContainerFactory defines the functionality to create an interceptors container type InterceptorsContainerFactory interface { Create() (InterceptorsContainer, InterceptorsContainer, error) + AddShardTrieNodeInterceptors(container InterceptorsContainer) error IsInterfaceNil() bool } diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index c7d005e94fb..b81c2b2b393 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -223,6 +223,11 @@ func (ficf *fullSyncInterceptorsContainerFactory) Create() (process.Interceptors return ficf.mainContainer, ficf.fullArchiveContainer, nil } +// AddShardTrieNodeInterceptors returns nil +func (ficf *fullSyncInterceptorsContainerFactory) AddShardTrieNodeInterceptors(_ process.InterceptorsContainer) error { + return nil +} + func checkBaseParams( coreComponents process.CoreComponentsHolder, cryptoComponents process.CryptoComponentsHolder, From 50d8964cfef51271c197793065fb0cecbdb19a30 Mon Sep 17 00:00:00 2001 From: radu Date: Wed, 13 May 2026 14:34:25 +0300 Subject: [PATCH 038/116] added check for minimum blocks per epoch in trigger --- epochStart/metachain/trigger.go | 9 ++-- epochStart/metachain/trigger_test.go | 75 +++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/epochStart/metachain/trigger.go b/epochStart/metachain/trigger.go index 9d4855bb11e..3aafa201090 100644 --- a/epochStart/metachain/trigger.go +++ b/epochStart/metachain/trigger.go @@ -15,13 +15,13 @@ import ( "github.com/multiversx/mx-chain-core-go/display" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("epochStart/metachain") @@ -32,7 +32,7 @@ var _ process.EpochStartTriggerHandler = (*trigger)(nil) var _ process.EpochBootstrapper = (*trigger)(nil) var _ closing.Closer = (*trigger)(nil) -const minimumNonceToStartEpoch = 4 +const minimumBlocksPerEpoch = 4 const disabledRoundForForceEpochStart = math.MaxUint64 // ArgsNewMetaEpochStartTrigger defines struct needed to create a new start of epoch trigger @@ -208,10 +208,11 @@ func (t *trigger) Update(round uint64, nonce uint64) { return } - isZeroEpochEdgeCase := nonce < minimumNonceToStartEpoch + epochStartNonce := t.epochStartMeta.GetNonce() + hasMinBlocksInEpoch := nonce >= epochStartNonce+minimumBlocksPerEpoch isNormalEpochStart := t.currentRound > t.currEpochStartRound+t.roundsPerEpoch isWithEarlyEndOfEpoch := t.currentRound >= t.nextEpochStartRound - shouldTriggerEpochStart := (isNormalEpochStart || isWithEarlyEndOfEpoch) && !isZeroEpochEdgeCase + shouldTriggerEpochStart := (isNormalEpochStart || isWithEarlyEndOfEpoch) && hasMinBlocksInEpoch if shouldTriggerEpochStart { t.epoch += 1 t.isEpochStart = true diff --git a/epochStart/metachain/trigger_test.go b/epochStart/metachain/trigger_test.go index c30a9cf4bd6..f88b2c6f1d9 100644 --- a/epochStart/metachain/trigger_test.go +++ b/epochStart/metachain/trigger_test.go @@ -257,12 +257,85 @@ func TestTrigger_ForceEpochStartShouldOk(t *testing.T) { assert.Equal(t, expectedRound, epochStartTrigger.nextEpochStartRound) - epochStartTrigger.Update(expectedRound, minimumNonceToStartEpoch) + epochStartTrigger.Update(expectedRound, minimumBlocksPerEpoch) isEpochStart := epochStartTrigger.IsEpochStart() assert.True(t, isEpochStart) } +func TestTrigger_ForceEpochStartShouldWaitMinimumNonceEvenWhenForced(t *testing.T) { + t.Parallel() + + arguments := createMockEpochStartTriggerArguments() + arguments.Settings.MinRoundsBetweenEpochs = 20 + arguments.Settings.RoundsPerEpoch = 200 + + epochStartTrigger, err := NewEpochStartTrigger(arguments) + require.Nil(t, err) + + forcedRound := uint64(60) + epochStartTrigger.ForceEpochStart(forcedRound) + + epochStartTrigger.Update(forcedRound, minimumBlocksPerEpoch-1) + assert.False(t, epochStartTrigger.IsEpochStart()) + + epochStartTrigger.Update(forcedRound, minimumBlocksPerEpoch) + assert.True(t, epochStartTrigger.IsEpochStart()) +} + +func TestTrigger_UpdateShouldWaitMinimumNonceFromPreviousEpochStart(t *testing.T) { + t.Parallel() + + arguments := createMockEpochStartTriggerArguments() + epochStartTrigger, err := NewEpochStartTrigger(arguments) + require.Nil(t, err) + + epochStartNonce := uint64(100) + epochStartTrigger.epochStartMeta = &block.MetaBlock{Nonce: epochStartNonce} + + round := uint64(3) + epochStartTrigger.Update(round, epochStartNonce+minimumBlocksPerEpoch-1) + assert.False(t, epochStartTrigger.IsEpochStart()) + + epochStartTrigger.Update(round, epochStartNonce+minimumBlocksPerEpoch) + assert.True(t, epochStartTrigger.IsEpochStart()) +} + +func TestTrigger_UpdateShouldEnforceMinBlocksAfterEpochTransition(t *testing.T) { + t.Parallel() + + arguments := createMockEpochStartTriggerArguments() + epochStartTrigger, err := NewEpochStartTrigger(arguments) + require.Nil(t, err) + + // default mock: RoundsPerEpoch=2, currEpochStartRound=0 + // round 3 > 0+2 satisfies isNormalEpochStart; nonce 4 satisfies hasMinBlocksInEpoch (4 >= 0+4) + epochStartTrigger.Update(3, minimumBlocksPerEpoch) + assert.True(t, epochStartTrigger.IsEpochStart()) + + // SetProcessed moves epochStartMeta to the epoch-1-start block at nonce 500 + // this resets the baseline: next epoch needs currentNonce >= 500+4 + epochOneStartNonce := uint64(500) + epochStartTrigger.SetProcessed(&block.MetaBlock{ + Round: 3, + Nonce: epochOneStartNonce, + Epoch: 1, + EpochStart: block.EpochStart{ + LastFinalizedHeaders: []block.EpochStartShardData{{RootHash: []byte("root")}}, + }, + }, nil) + assert.False(t, epochStartTrigger.IsEpochStart()) + + // round 6 > 3+2 satisfies isNormalEpochStart for epoch 2 + // but only 3 blocks since epoch 1 start - hasMinBlocksInEpoch must block it + epochStartTrigger.Update(6, epochOneStartNonce+minimumBlocksPerEpoch-1) + assert.False(t, epochStartTrigger.IsEpochStart()) + + // 4th block since epoch 1 start - guard satisfied + epochStartTrigger.Update(6, epochOneStartNonce+minimumBlocksPerEpoch) + assert.True(t, epochStartTrigger.IsEpochStart()) +} + func TestTrigger_LastCommitedMetaEpochStartBlock(t *testing.T) { t.Parallel() From 50c42b67e344b4085d6ddb6e0f22eaf00e10e9a8 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 13 May 2026 14:57:50 +0300 Subject: [PATCH 039/116] fixes after review --- consensus/broadcast/delayedBroadcast.go | 40 ++++++++++++------------- consensus/broadcast/export.go | 4 +-- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/consensus/broadcast/delayedBroadcast.go b/consensus/broadcast/delayedBroadcast.go index 49087024ada..5f092b06ef1 100644 --- a/consensus/broadcast/delayedBroadcast.go +++ b/consensus/broadcast/delayedBroadcast.go @@ -83,7 +83,7 @@ type delayedBlockBroadcaster struct { // pendingMetaHeaders stores metachain headers waiting for proof arrival before broadcast. // mutPendingMetaHeaders and mutDataForBroadcast are never held simultaneously. pendingMetaHeaders map[string]*pendingHeaderInfo - mutPendingMetaHeaders sync.Mutex + mutPendingMetaHeaders sync.RWMutex cacheProcessedMetaHeaders storage.Cacher } @@ -299,18 +299,13 @@ func (dbb *delayedBlockBroadcaster) headerReceived(headerHandler data.HeaderHand return } - if common.IsProofsFlagEnabledForHeader(dbb.enableEpochsHandler, headerHandler) { - if !dbb.proofsPool.HasProof(headerHandler.GetShardID(), headerHash) { - dbb.addPendingMetaHeader(headerHandler, headerHash) - log.Trace("delayedBlockBroadcaster.headerReceived: proof not yet available, deferring broadcast", - "headerHash", headerHash, - "nonce", headerHandler.GetNonce(), - ) - return - } + if !common.IsProofsFlagEnabledForHeader(dbb.enableEpochsHandler, headerHandler) { + dbb.processMetachainHeader(headerHandler, headerHash) + return } - dbb.processMetachainHeader(headerHandler, headerHash) + dbb.addPendingMetaHeader(headerHandler, headerHash) + dbb.tryProcessPendingMetaHeader(headerHash) } func (dbb *delayedBlockBroadcaster) proofReceived(proof data.HeaderProofHandler) { @@ -322,24 +317,27 @@ func (dbb *delayedBlockBroadcaster) proofReceived(proof data.HeaderProofHandler) } headerHash := proof.GetHeaderHash() - hashStr := string(headerHash) + dbb.tryProcessPendingMetaHeader(headerHash) dbb.mutPendingMetaHeaders.Lock() - pending, found := dbb.pendingMetaHeaders[hashStr] - if found { - delete(dbb.pendingMetaHeaders, hashStr) - } dbb.evictPendingMetaHeadersUpToNonce(proof.GetHeaderNonce()) dbb.mutPendingMetaHeaders.Unlock() +} +func (dbb *delayedBlockBroadcaster) tryProcessPendingMetaHeader(headerHash []byte) { + dbb.mutPendingMetaHeaders.Lock() + hashStr := string(headerHash) + pending, found := dbb.pendingMetaHeaders[hashStr] if !found { + dbb.mutPendingMetaHeaders.Unlock() return } - - log.Trace("delayedBlockBroadcaster.proofReceived: proof arrived, triggering deferred broadcast", - "headerHash", headerHash, - "nonce", pending.nonce, - ) + if !dbb.proofsPool.HasProof(core.MetachainShardId, headerHash) { + dbb.mutPendingMetaHeaders.Unlock() + return + } + delete(dbb.pendingMetaHeaders, hashStr) + dbb.mutPendingMetaHeaders.Unlock() dbb.processMetachainHeader(pending.header, pending.hash) } diff --git a/consensus/broadcast/export.go b/consensus/broadcast/export.go index fbde6330f1a..0aded0e0fce 100644 --- a/consensus/broadcast/export.go +++ b/consensus/broadcast/export.go @@ -88,8 +88,8 @@ func (dbb *delayedBlockBroadcaster) ProofReceived(proof data.HeaderProofHandler) // GetPendingMetaHeadersCount returns the number of pending meta headers func (dbb *delayedBlockBroadcaster) GetPendingMetaHeadersCount() int { - dbb.mutPendingMetaHeaders.Lock() - defer dbb.mutPendingMetaHeaders.Unlock() + dbb.mutPendingMetaHeaders.RLock() + defer dbb.mutPendingMetaHeaders.RUnlock() return len(dbb.pendingMetaHeaders) } From 7978b619aaca66a8f6a2a54673ae9348cadd8286 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 13 May 2026 16:40:14 +0300 Subject: [PATCH 040/116] mb check return err on fail --- consensus/spos/errors.go | 3 ++ consensus/spos/worker.go | 19 ++++++---- consensus/spos/worker_test.go | 66 ++++++++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/consensus/spos/errors.go b/consensus/spos/errors.go index d89a58865f3..4a8c53db18e 100644 --- a/consensus/spos/errors.go +++ b/consensus/spos/errors.go @@ -121,6 +121,9 @@ var ErrInvalidSignature = errors.New("signature is invalid") // ErrInvalidHeader is raised when header is invalid var ErrInvalidHeader = errors.New("header is invalid") +// ErrInvalidBody is raised when body is invalid +var ErrInvalidBody = errors.New("body is invalid") + // ErrMessageFromItself is raised when a message from itself is received var ErrMessageFromItself = errors.New("message is from itself") diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 9aa4db326b7..bc81ce05d3a 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -528,7 +528,10 @@ func (wrk *Worker) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedP isMessageWithInvalidSigners := wrk.consensusService.IsMessageWithInvalidSigners(msgType) if isMessageWithBlockBody || isMessageWithBlockBodyAndHeader { - wrk.doJobOnMessageWithBlockBody(cnsMsg) + err = wrk.doJobOnMessageWithBlockBody(cnsMsg) + if err != nil { + return nil, err + } } if isMessageWithBlockHeader || isMessageWithBlockBodyAndHeader { @@ -579,8 +582,8 @@ func (wrk *Worker) shouldBlacklistPeer(err error) bool { return true } -func (wrk *Worker) doJobOnMessageWithBlockBody(cnsMsg *consensus.Message) { - wrk.addBlockToPool(cnsMsg.GetBody()) +func (wrk *Worker) doJobOnMessageWithBlockBody(cnsMsg *consensus.Message) error { + return wrk.addBlockToPool(cnsMsg.GetBody()) } func (wrk *Worker) doJobOnMessageWithHeader(cnsMsg *consensus.Message) error { @@ -676,28 +679,30 @@ func (wrk *Worker) doJobOnMessageWithSignature(cnsMsg *consensus.Message, p2pMsg ) } -func (wrk *Worker) addBlockToPool(bodyBytes []byte) { +func (wrk *Worker) addBlockToPool(bodyBytes []byte) error { bodyHandler := wrk.blockProcessor.DecodeBlockBody(bodyBytes) body, ok := bodyHandler.(*block.Body) if !ok { - return + return ErrInvalidBody } for _, miniblock := range body.MiniBlocks { err := process.CheckMiniBlock(miniblock, wrk.shardCoordinator) if err != nil { log.Debug("addBlockToPool: invalid miniblock in received consensus body", "error", err.Error()) - return + return err } } for _, miniblock := range body.MiniBlocks { hash, err := core.CalculateHash(wrk.marshalizer, wrk.hasher, miniblock) if err != nil { - return + return err } wrk.poolAdder.Put(hash, miniblock, miniblock.Size()) } + + return nil } func (wrk *Worker) processReceivedHeaderMetricForConsensusMessage(cnsDta *consensus.Message) { diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index a1eec9d7525..e3c8201cb7b 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -60,7 +60,7 @@ func createDefaultWorkerArgs(appStatusHandler core.AppStatusHandler) *spos.Worke RevertCurrentBlockCalled: func() { }, DecodeBlockBodyCalled: func(dta []byte) data.BodyHandler { - return nil + return &block.Body{} }, } bootstrapperMock := &bootstrapperStubs.BootstrapperStub{} @@ -563,6 +563,7 @@ func TestWorker_RemoveAllReceivedMessageCallsShouldWork(t *testing.T) { func TestWorker_ProcessReceivedMessageTxBlockBodyShouldRetNil(t *testing.T) { t.Parallel() + wrk := *initWorker(&statusHandlerMock.AppStatusHandlerStub{}) blk := &block.Body{} blkStr, _ := mock.MarshalizerMock{}.Marshal(blk) @@ -584,16 +585,79 @@ func TestWorker_ProcessReceivedMessageTxBlockBodyShouldRetNil(t *testing.T) { ) buff, _ := wrk.Marshalizer().Marshal(cnsMsg) time.Sleep(time.Second) + msg := &p2pmocks.P2PMessageMock{ DataField: buff, PeerField: currentPid, SignatureField: []byte("signature"), } + msgID, err := wrk.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) assert.Nil(t, err) assert.Len(t, msgID, 0) } +func TestWorker_ProcessReceivedMessage_InvalidBody_ShouldFail(t *testing.T) { + t.Parallel() + + blk := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + &block.MiniBlock{ + SenderShardID: 1, + ReceiverShardID: 2, + Type: block.TxBlock, + }, + &block.MiniBlock{ + SenderShardID: 1, // invalid sender shard id + ReceiverShardID: 0, + Type: block.RewardsBlock, + }, + }, + } + blkStr, _ := mock.MarshalizerMock{}.Marshal(blk) + + blockProcessor := &testscommon.BlockProcessorStub{ + DecodeBlockBodyCalled: func(dta []byte) data.BodyHandler { + return blk + }, + } + + workerArgs := createDefaultWorkerArgs(&statusHandlerMock.AppStatusHandlerStub{}) + workerArgs.BlockProcessor = blockProcessor + wrk, _ := spos.NewWorker(workerArgs) + + wrk.ConsensusState().SetHeader(&block.HeaderV2{}) + + cnsMsg := consensus.NewConsensusMessage( + nil, + nil, + blkStr, + nil, + []byte(wrk.ConsensusState().ConsensusGroup()[0]), + signature, + int(bls.MtBlockBody), + 0, + chainID, + nil, + nil, + nil, + currentPid, + nil, + ) + buff, _ := wrk.Marshalizer().Marshal(cnsMsg) + time.Sleep(time.Second) + + msg := &p2pmocks.P2PMessageMock{ + DataField: buff, + PeerField: currentPid, + SignatureField: []byte("signature"), + } + + msgID, err := wrk.ProcessReceivedMessage(msg, fromConnectedPeerId, &p2pmocks.MessengerStub{}) + assert.ErrorIs(t, err, process.ErrInvalidShardId) + assert.Len(t, msgID, 0) +} + func TestWorker_ProcessReceivedMessageNilMessageShouldErr(t *testing.T) { t.Parallel() wrk := *initWorker(&statusHandlerMock.AppStatusHandlerStub{}) From 94cd7734ab63cef203877d6be757e40fb7d83238 Mon Sep 17 00:00:00 2001 From: miiu Date: Thu, 14 May 2026 10:56:14 +0300 Subject: [PATCH 041/116] fixes after review --- cmd/node/config/config.toml | 66 ++++++++++++++--------------- config/config.go | 4 +- go.mod | 2 +- go.sum | 4 +- storage/factory/persisterCreator.go | 12 +++--- 5 files changed, 44 insertions(+), 44 deletions(-) diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 82401e8077b..fbc940bb719 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -119,7 +119,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [ReceiptsStorage] [ReceiptsStorage.Cache] @@ -133,7 +133,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [ScheduledSCRsStorage] [ScheduledSCRsStorage.Cache] @@ -147,7 +147,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [PeerBlockBodyStorage] [PeerBlockBodyStorage.Cache] @@ -161,7 +161,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [BlockHeaderStorage] [BlockHeaderStorage.Cache] @@ -175,7 +175,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [BootstrapStorage] [BootstrapStorage.Cache] @@ -189,7 +189,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [MetaBlockStorage] [MetaBlockStorage.Cache] @@ -203,7 +203,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [ProofsStorage] [ProofsStorage.Cache] @@ -217,7 +217,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [TxStorage] [TxStorage.Cache] @@ -231,7 +231,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 30000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [UnsignedTransactionStorage] [UnsignedTransactionStorage.Cache] @@ -245,7 +245,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [RewardTxStorage] [RewardTxStorage.Cache] @@ -259,7 +259,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [SmartContractsStorage] [SmartContractsStorage.Cache] @@ -273,7 +273,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [SmartContractsStorageSimulate] [SmartContractsStorageSimulate.Cache] @@ -287,7 +287,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [SmartContractsStorageForSCQuery] [SmartContractsStorageForSCQuery.Cache] @@ -301,7 +301,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [StatusMetricsStorage] [StatusMetricsStorage.Cache] @@ -314,7 +314,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [TrieEpochRootHashStorage] [TrieEpochRootHashStorage.Cache] @@ -328,7 +328,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 500 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [ShardHdrNonceHashStorage] [ShardHdrNonceHashStorage.Cache] @@ -342,7 +342,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [MetaHdrNonceHashStorage] [MetaHdrNonceHashStorage.Cache] @@ -356,7 +356,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [AccountsTrieStorage] [AccountsTrieStorage.Cache] @@ -385,7 +385,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [PeerAccountsTrieStorage] [PeerAccountsTrieStorage.Cache] @@ -412,7 +412,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [TrieStorageManagerConfig] PruningBufferLen = 100000 @@ -550,7 +550,7 @@ MaxBatchSize = 45000 MaxOpenFiles = 10 UseTmpAsFilePath = true - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Antiflood] Enabled = true @@ -801,7 +801,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Hardfork.ExportKeysStorageConfig] [Hardfork.ExportKeysStorageConfig.Cache] Name = "HardFork.ExportKeysStorageConfig" @@ -813,7 +813,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Hardfork.ExportTriesStorageConfig] [Hardfork.ExportTriesStorageConfig.Cache] Name = "HardFork.ExportTriesStorageConfig" @@ -825,7 +825,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Hardfork.ImportStateStorageConfig] [Hardfork.ImportStateStorageConfig.Cache] Name = "HardFork.ImportStateStorageConfig" @@ -837,7 +837,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Hardfork.ImportKeysStorageConfig] [Hardfork.ImportKeysStorageConfig.Cache] Name = "HardFork.ImportKeysStorageConfig" @@ -849,7 +849,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 1000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Debug] [Debug.InterceptorResolver] @@ -908,7 +908,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 100 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [DbLookupExtensions] Enabled = false @@ -923,7 +923,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [DbLookupExtensions.MiniblockHashByTxHashStorageConfig.Cache] Name = "DbLookupExtensions.MiniblockHashByTxHashStorage" Capacity = 20000 @@ -934,7 +934,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [DbLookupExtensions.EpochByHashStorageConfig.Cache] Name = "DbLookupExtensions.EpochByHashStorage" Capacity = 20000 @@ -945,7 +945,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [DbLookupExtensions.ResultsHashesByTxHashStorageConfig.Cache] Name = "DbLookupExtensions.ResultsHashesByTxHashStorage" Capacity = 20000 @@ -956,7 +956,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [DbLookupExtensions.ESDTSuppliesStorageConfig.Cache] Name = "DbLookupExtensions.ESDTSuppliesStorage" Capacity = 20000 @@ -967,7 +967,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [DbLookupExtensions.RoundHashStorageConfig.Cache] Name = "DbLookupExtensions.RoundHashStorage" Capacity = 20000 @@ -978,7 +978,7 @@ BatchDelaySeconds = 2 MaxBatchSize = 20000 MaxOpenFiles = 10 - BloomFilterBtsPerKey = 10 + BloomFilterBitsPerKey = 10 [Logs] LogFileLifeSpanInMB = 1024 # 1GB diff --git a/config/config.go b/config/config.go index 3f57c220cba..9be5da29eb5 100644 --- a/config/config.go +++ b/config/config.go @@ -35,9 +35,9 @@ type DBConfig struct { UseTmpAsFilePath bool ShardIDProviderType string NumShards int32 - // BloomFilterBtsPerKey == 0, the Bloom filter is disabled. + // BloomFilterBitsPerKey == 0, the Bloom filter is disabled. // Otherwise, it specifies the number of bits per key used by the Bloom filter. - BloomFilterBtsPerKey int + BloomFilterBitsPerKey int } // StorageConfig will map the storage unit configuration diff --git a/go.mod b/go.mod index ff7452059b8..258c0d03821 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.9.2 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 - github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260512115600-4b656efb29e0 + github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260514073036-7edefb9fa687 github.com/multiversx/mx-chain-vm-common-go v1.6.5 github.com/multiversx/mx-chain-vm-go v1.5.45 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 diff --git a/go.sum b/go.sum index e876af0866c..95144f461b5 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260512115600-4b656efb29e0 h1:S4H5Vhq2bdC6TtZuMKdcPvD+vJ2BfSxkcfvO/l43buA= -github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260512115600-4b656efb29e0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260514073036-7edefb9fa687 h1:JOm631MmbFkI1+43bp4DKT+ywfPJGaiExmdDY5RpvEI= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260514073036-7edefb9fa687/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= diff --git a/storage/factory/persisterCreator.go b/storage/factory/persisterCreator.go index c9f49eeb72e..a1fc19cd3e8 100644 --- a/storage/factory/persisterCreator.go +++ b/storage/factory/persisterCreator.go @@ -43,12 +43,12 @@ func (pc *persisterCreator) CreateBasePersister(path string) (storage.Persister, var dbType = storageunit.DBType(pc.conf.Type) argsDB := factory.ArgDB{ - DBType: dbType, - Path: path, - BatchDelaySeconds: pc.conf.BatchDelaySeconds, - MaxBatchSize: pc.conf.MaxBatchSize, - MaxOpenFiles: pc.conf.MaxOpenFiles, - BloomFilterBtsPerKey: pc.conf.BloomFilterBtsPerKey, + DBType: dbType, + Path: path, + BatchDelaySeconds: pc.conf.BatchDelaySeconds, + MaxBatchSize: pc.conf.MaxBatchSize, + MaxOpenFiles: pc.conf.MaxOpenFiles, + BloomFilterBitsPerKey: pc.conf.BloomFilterBitsPerKey, } return storageunit.NewDB(argsDB) From 470c79f34886746345134226c05d710bd09afdb0 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 14 May 2026 11:24:34 +0300 Subject: [PATCH 042/116] concurrent checks + test --- consensus/spos/consensusState.go | 9 +++++ consensus/spos/worker.go | 12 ++++++- consensus/spos/worker_test.go | 61 ++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/consensus/spos/consensusState.go b/consensus/spos/consensusState.go index 476b90133e6..a427b5277c3 100644 --- a/consensus/spos/consensusState.go +++ b/consensus/spos/consensusState.go @@ -70,9 +70,12 @@ func NewConsensusState( // ResetConsensusRoundState method resets all the consensus round data (except messages received) func (cns *ConsensusState) ResetConsensusRoundState() { + cns.mutState.Lock() cns.RoundCanceled = false cns.ExtendedCalled = false cns.WaitingAllSignaturesTimeOut = false + cns.mutState.Unlock() + cns.ResetRoundStatus() cns.ResetRoundState() } @@ -444,11 +447,17 @@ func (cns *ConsensusState) SetRoundTimeStamp(roundTimeStamp time.Time) { // GetExtendedCalled returns the state of the extended called func (cns *ConsensusState) GetExtendedCalled() bool { + cns.mutState.RLock() + defer cns.mutState.RUnlock() + return cns.ExtendedCalled } // SetExtendedCalled sets the state of the extended called func (cns *ConsensusState) SetExtendedCalled(extendedCalled bool) { + cns.mutState.Lock() + defer cns.mutState.Unlock() + cns.ExtendedCalled = extendedCalled } diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index bc81ce05d3a..842e214d251 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -83,7 +83,9 @@ type Worker struct { antifloodHandler consensus.P2PAntifloodHandler poolAdder PoolAdder - cancelFunc func() + cancelFunc func() + mutWorker sync.RWMutex + consensusMessageValidator *consensusMessageValidator nodeRedundancyHandler consensus.NodeRedundancyHandler peerBlacklistHandler consensus.PeerBlacklistHandler @@ -197,7 +199,11 @@ func NewWorker(args *WorkerArgs) (*Worker, error) { // StartWorking actually starts the consensus working mechanism func (wrk *Worker) StartWorking() { var ctx context.Context + + wrk.mutWorker.Lock() ctx, wrk.cancelFunc = context.WithCancel(context.Background()) + wrk.mutWorker.Unlock() + go wrk.checkChannels(ctx) } @@ -818,6 +824,7 @@ func (wrk *Worker) checkChannels(ctx context.Context) { msgType := consensus.MessageType(rcvDta.MsgType) + wrk.mutReceivedMessagesCalls.RLock() if receivedMessageCallbacks, exist := wrk.receivedMessagesCalls[msgType]; exist { for _, callReceivedMessage := range receivedMessageCallbacks { if callReceivedMessage(ctx, rcvDta) { @@ -828,6 +835,7 @@ func (wrk *Worker) checkChannels(ctx context.Context) { } } } + wrk.mutReceivedMessagesCalls.RUnlock() wrk.callReceivedHeaderCallbacks(rcvDta) } @@ -943,9 +951,11 @@ func (wrk *Worker) Close() error { // (just to close some go routines started as edge cases that would otherwise hang) defer wrk.closer.Close() + wrk.mutWorker.RLock() if wrk.cancelFunc != nil { wrk.cancelFunc() } + wrk.mutWorker.RUnlock() wrk.cleanChannels() diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index e3c8201cb7b..60e7090c3d7 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math/big" "strconv" + "sync" "sync/atomic" "testing" "time" @@ -2497,3 +2498,63 @@ func TestWorker_ReceivedProof(t *testing.T) { require.True(t, wasHandlerCalled) }) } + +func TestWorker_Concurrency(t *testing.T) { + t.Parallel() + + workerArgs := createDefaultWorkerArgs(&statusHandlerMock.AppStatusHandlerStub{}) + wrk, _ := spos.NewWorker(workerArgs) + + wg := sync.WaitGroup{} + + numOperations := 500 + wg.Add(numOperations) + + for i := 0; i < numOperations; i++ { + go func(idx int) { + switch idx { + case 0: + wrk.AddReceivedHeaderHandler(func(handler data.HeaderHandler) {}) + case 1: + wrk.AddReceivedMessageCall(bls.MtBlockBody, nil) + case 2: + wrk.AddReceivedProofHandler(func(proof consensus.ProofHandler) {}) + case 3: + _ = wrk.Close() + case 4: + wrk.DisplayStatistics() + case 5: + wrk.ExecuteStoredMessages() + case 6: + wrk.Extend(0) + case 7: + _ = wrk.GetConsensusStateChangedChannel() + case 8: + _, _ = wrk.ProcessReceivedMessage(&p2pmocks.P2PMessageMock{}, fromConnectedPeerId, &p2pmocks.MessengerStub{}) + case 9: + wrk.ReceivedHeader(&block.Header{ + ShardID: workerArgs.ShardCoordinator.SelfId(), + Round: uint64(workerArgs.RoundHandler.Index()), + }, nil) + case 10: + wrk.ReceivedProof(&block.HeaderProof{}) + case 11: + wrk.RemoveAllReceivedHeaderHandlers() + case 12: + wrk.RemoveAllReceivedMessagesCalls() + case 13: + wrk.ResetConsensusMessages() + case 14: + wrk.ResetConsensusRoundState() + case 15: + wrk.ResetInvalidSignersCache() + case 16: + wrk.StartWorking() + } + + wg.Done() + }(i % 17) + } + + wg.Wait() +} From 392840b1531e3c563bd502be8c2ae509130eeca8 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 14 May 2026 11:36:26 +0300 Subject: [PATCH 043/116] fix after review --- consensus/spos/consensusMessageValidator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/spos/consensusMessageValidator.go b/consensus/spos/consensusMessageValidator.go index 5bc6bc52520..54b97a03f57 100644 --- a/consensus/spos/consensusMessageValidator.go +++ b/consensus/spos/consensusMessageValidator.go @@ -531,8 +531,8 @@ func (cmv *consensusMessageValidator) addMessageTypeToPublicKey(pk []byte, round } func (cmv *consensusMessageValidator) removeMessageTypeToPublicKey(pk []byte, round int64, msgType consensus.MessageType) { - cmv.mutPkConsensusMessages.RLock() - defer cmv.mutPkConsensusMessages.RUnlock() + cmv.mutPkConsensusMessages.Lock() + defer cmv.mutPkConsensusMessages.Unlock() key := fmt.Sprintf("%s_%d", string(pk), round) From 436c64c4c314040a8ca22db500f0170fbfe24f27 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 14 May 2026 13:08:41 +0300 Subject: [PATCH 044/116] update test --- consensus/spos/worker_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consensus/spos/worker_test.go b/consensus/spos/worker_test.go index 60e7090c3d7..b09039e5de1 100644 --- a/consensus/spos/worker_test.go +++ b/consensus/spos/worker_test.go @@ -2550,6 +2550,8 @@ func TestWorker_Concurrency(t *testing.T) { wrk.ResetInvalidSignersCache() case 16: wrk.StartWorking() + default: + require.Fail(t, "should have not been called") } wg.Done() From 7d7996b1cc4ae5620efedb6cdbed3130b632d96b Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 14 May 2026 16:06:13 +0300 Subject: [PATCH 045/116] fixes after review --- consensus/spos/worker.go | 6 ++++-- process/common_test.go | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 842e214d251..6914d49714d 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -825,7 +825,10 @@ func (wrk *Worker) checkChannels(ctx context.Context) { msgType := consensus.MessageType(rcvDta.MsgType) wrk.mutReceivedMessagesCalls.RLock() - if receivedMessageCallbacks, exist := wrk.receivedMessagesCalls[msgType]; exist { + receivedMessageCallbacks, exist := wrk.receivedMessagesCalls[msgType] + wrk.mutReceivedMessagesCalls.RUnlock() + + if exist { for _, callReceivedMessage := range receivedMessageCallbacks { if callReceivedMessage(ctx, rcvDta) { select { @@ -835,7 +838,6 @@ func (wrk *Worker) checkChannels(ctx context.Context) { } } } - wrk.mutReceivedMessagesCalls.RUnlock() wrk.callReceivedHeaderCallbacks(rcvDta) } diff --git a/process/common_test.go b/process/common_test.go index f34198376db..8249ef0b1da 100644 --- a/process/common_test.go +++ b/process/common_test.go @@ -2511,6 +2511,10 @@ func TestCheckMiniBlock(t *testing.T) { mb := &block.MiniBlock{SenderShardID: wrongShardId, ReceiverShardID: 1, Type: block.TxBlock} err := process.CheckMiniBlock(mb, shardCoordinator) require.ErrorIs(t, err, process.ErrInvalidShardId) + + mb = &block.MiniBlock{SenderShardID: core.AllShardId, ReceiverShardID: 1, Type: block.TxBlock} + err = process.CheckMiniBlock(mb, shardCoordinator) + require.ErrorIs(t, err, process.ErrInvalidShardId) }) t.Run("nil tx hash, should fail", func(t *testing.T) { From 95302cba43c1b2d723c0936b6b9951010925dabb Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 14 May 2026 16:27:45 +0300 Subject: [PATCH 046/116] update shard id check vars --- process/common.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/process/common.go b/process/common.go index 7764ac68103..3f0416f7682 100644 --- a/process/common.go +++ b/process/common.go @@ -1145,33 +1145,36 @@ func CheckMiniBlock( miniBlock *block.MiniBlock, shardCoordinator sharding.Coordinator, ) error { + senderShard := miniBlock.GetSenderShardID() + receiverShard := miniBlock.GetReceiverShardID() + // shard id checks - receiverNotCurrentShard := miniBlock.ReceiverShardID >= shardCoordinator.NumberOfShards() && - (miniBlock.ReceiverShardID != core.MetachainShardId && miniBlock.ReceiverShardID != core.AllShardId) - if receiverNotCurrentShard { + receiverShardInvalid := receiverShard >= shardCoordinator.NumberOfShards() && + (receiverShard != core.MetachainShardId && receiverShard != core.AllShardId) + if receiverShardInvalid { return fmt.Errorf("%w - receiver not for current shard: block type: %s, sender shard id: %d, receiver shard id: %d", ErrInvalidShardId, miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) + senderShard, + receiverShard) } - senderNotCurrentShard := miniBlock.SenderShardID >= shardCoordinator.NumberOfShards() && - miniBlock.SenderShardID != core.MetachainShardId - if senderNotCurrentShard { + senderShardInvalid := senderShard >= shardCoordinator.NumberOfShards() && + senderShard != core.MetachainShardId + if senderShardInvalid { return fmt.Errorf("%w - sender not for current shard: block type: %s, sender shard id: %d, receiver shard id: %d", ErrInvalidShardId, miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) + senderShard, + receiverShard) } - if miniBlock.SenderShardID != shardCoordinator.SelfId() && miniBlock.GetReceiverShardID() != shardCoordinator.SelfId() && miniBlock.GetReceiverShardID() != core.AllShardId { + if senderShard != shardCoordinator.SelfId() && receiverShard != shardCoordinator.SelfId() && receiverShard != core.AllShardId { return fmt.Errorf("%w - not valid shard ids: block type: %s, sender shard id: %d, receiver shard id: %d", ErrInvalidShardId, miniBlock.Type, - miniBlock.SenderShardID, - miniBlock.ReceiverShardID) + senderShard, + receiverShard) } err := checkMiniBlockByType(miniBlock, shardCoordinator) From a5c613a70e703afa0e38be878b5f613afb73d621 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 14 May 2026 17:09:02 +0300 Subject: [PATCH 047/116] fix after review --- consensus/spos/consensusMessageValidator.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/consensus/spos/consensusMessageValidator.go b/consensus/spos/consensusMessageValidator.go index 54b97a03f57..2c344b10659 100644 --- a/consensus/spos/consensusMessageValidator.go +++ b/consensus/spos/consensusMessageValidator.go @@ -500,16 +500,18 @@ func (cmv *consensusMessageValidator) isMessageTypeLimitReached(pk []byte, round mapMsgType, ok := cmv.mapPkConsensusMessages[key] if !ok { - cmv.addMessageTypeToPublicKey(pk, round, msgType) - return false + return cmv.checkLimitReached(0, pk, round, msgType) } numMsgType, ok := mapMsgType[msgType] if !ok { - cmv.addMessageTypeToPublicKey(pk, round, msgType) - return false + return cmv.checkLimitReached(numMsgType, pk, round, msgType) } + return cmv.checkLimitReached(numMsgType, pk, round, msgType) +} + +func (cmv *consensusMessageValidator) checkLimitReached(numMsgType uint32, pk []byte, round int64, msgType consensus.MessageType) bool { isLimitReached := numMsgType >= cmv.consensusService.GetMaxNumOfMessageTypeAccepted(msgType) if !isLimitReached { cmv.addMessageTypeToPublicKey(pk, round, msgType) From cd35f24239753702ac6bdf4095d5d854d31aaec4 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 14 May 2026 18:05:00 +0300 Subject: [PATCH 048/116] fix after review --- process/interceptors/multiDataInterceptor.go | 4 +--- process/interceptors/multiDataInterceptor_test.go | 6 +++--- process/interceptors/singleDataInterceptor.go | 4 +--- process/interceptors/singleDataInterceptor_test.go | 6 +++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 643457286d8..054a57a0dbe 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -235,15 +235,13 @@ func (mdi *MultiDataInterceptor) interceptedData( } isForCurrentShard := interceptedData.IsForCurrentShard() - shouldProcess := isForCurrentShard || isWhiteListed - if !shouldProcess { + if !isForCurrentShard { log.Trace("intercepted data should not be processed", "pid", p2p.MessageOriginatorPid(message), "seq no", p2p.MessageOriginatorSeq(message), "topic", message.Topic(), "hash", interceptedData.Hash(), "is for this shard", isForCurrentShard, - "is white listed", isWhiteListed, ) return nil, process.ErrInterceptedDataNotForCurrentShard } diff --git a/process/interceptors/multiDataInterceptor_test.go b/process/interceptors/multiDataInterceptor_test.go index 3f0e303af1c..22e3005c105 100644 --- a/process/interceptors/multiDataInterceptor_test.go +++ b/process/interceptors/multiDataInterceptor_test.go @@ -555,7 +555,7 @@ func TestMultiDataInterceptor_ProcessReceivedMessageWhitelistedShouldRetNil(t *t return nil }, IsForCurrentShardCalled: func() bool { - return false + return true }, HashCalled: func() []byte { return msgHash @@ -618,7 +618,7 @@ func processReceivedMessageMultiDataInvalidVersion(t *testing.T, expectedErr err return expectedErr }, IsForCurrentShardCalled: func() bool { - return false + return true }, } @@ -707,7 +707,7 @@ func TestMultiDataInterceptor_ProcessReceivedMessageIsOriginatorNotOkButWhiteLis return nil }, IsForCurrentShardCalled: func() bool { - return false + return true }, HashCalled: func() []byte { return msgHash diff --git a/process/interceptors/singleDataInterceptor.go b/process/interceptors/singleDataInterceptor.go index 8d0f1075fa0..7a0481eae00 100644 --- a/process/interceptors/singleDataInterceptor.go +++ b/process/interceptors/singleDataInterceptor.go @@ -114,8 +114,7 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, messageID := interceptedData.Hash() isForCurrentShard := interceptedData.IsForCurrentShard() - shouldProcess := isForCurrentShard || isWhiteListed - if !shouldProcess { + if !isForCurrentShard { sdi.throttler.EndProcessing() log.Trace("intercepted data is for other shards", "pid", p2p.MessageOriginatorPid(message), @@ -123,7 +122,6 @@ func (sdi *SingleDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, "topic", message.Topic(), "hash", interceptedData.Hash(), "is for current shard", isForCurrentShard, - "is white listed", isWhiteListed, ) return messageID, process.ErrInterceptedDataNotForCurrentShard diff --git a/process/interceptors/singleDataInterceptor_test.go b/process/interceptors/singleDataInterceptor_test.go index d64ee682359..e0176a294a1 100644 --- a/process/interceptors/singleDataInterceptor_test.go +++ b/process/interceptors/singleDataInterceptor_test.go @@ -297,7 +297,7 @@ func TestSingleDataInterceptor_ProcessReceivedMessageWhitelistedShouldWork(t *te return nil }, IsForCurrentShardCalled: func() bool { - return false + return true }, HashCalled: func() []byte { return msgHash @@ -355,7 +355,7 @@ func processReceivedMessageSingleDataInvalidVersion(t *testing.T, expectedErr er return expectedErr }, IsForCurrentShardCalled: func() bool { - return false + return true }, } @@ -411,7 +411,7 @@ func TestSingleDataInterceptor_ProcessReceivedMessageWithOriginator(t *testing.T return nil }, IsForCurrentShardCalled: func() bool { - return false + return true }, HashCalled: func() []byte { return msgHash From b2a731b2e313e5f65817af84ee88d858dea9e2ab Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 15 May 2026 13:11:54 +0300 Subject: [PATCH 049/116] integrate storage-go --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3113541dea9..fc8f5fd7317 100644 --- a/go.mod +++ b/go.mod @@ -208,3 +208,5 @@ require ( ) replace github.com/gogo/protobuf => github.com/multiversx/protobuf v1.3.2 + +replace github.com/multiversx/mx-chain-storage-go => github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260513084840-22d627716370 diff --git a/go.sum b/go.sum index b1920975338..b7cbf032404 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= -github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260513084840-22d627716370 h1:qhgeRSVZsnQqS7MBoN4XCpshKlVpXSS4fv/g49uNHJQ= +github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260513084840-22d627716370/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= From ba0e4e581e7560845b4b8080654a3051973b6560 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 15 May 2026 15:13:53 +0300 Subject: [PATCH 050/116] vm common with fixes --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 70cb5650b0a..3fb4665fcb9 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,8 @@ require ( gopkg.in/go-playground/validator.v8 v8.18.2 ) +replace github.com/multiversx/mx-chain-vm-common-go v1.6.6 => github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a + require ( github.com/TwiN/go-color v1.1.0 // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect diff --git a/go.sum b/go.sum index f5a56769576..5e48feb4256 100644 --- a/go.sum +++ b/go.sum @@ -413,8 +413,8 @@ github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4 github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= -github.com/multiversx/mx-chain-vm-common-go v1.6.6 h1:BJSQndP8KSqcSIi47wQwQy3uBIn5rbT3213eJroVaog= -github.com/multiversx/mx-chain-vm-common-go v1.6.6/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= +github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a h1:arc/Q+8Q8F1GnCGCtJu+sWnxzBmPgqtiIdzNk6tJ1T8= +github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= github.com/multiversx/mx-chain-vm-go v1.5.45/go.mod h1:Qc2Sckw+EfQwnapkzghFfhuUAOGv29oSZgvj8LJ+xWQ= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 h1:5gSR3IMw1mcp/v5oO+vZ5YOyWO8w7O2qKhCKNPwsWNE= From ea9b3b0da9b3b0a18f99de2d45ed3153edccf1c1 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 15 May 2026 15:16:52 +0300 Subject: [PATCH 051/116] go mod tidy --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 5e48feb4256..e075f63105f 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= -github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260514073036-7edefb9fa687 h1:JOm631MmbFkI1+43bp4DKT+ywfPJGaiExmdDY5RpvEI= +github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260514073036-7edefb9fa687/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a h1:arc/Q+8Q8F1GnCGCtJu+sWnxzBmPgqtiIdzNk6tJ1T8= github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= From bc77e55a385f31d5a9a95c221f13efb170c9cc01 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Fri, 15 May 2026 18:21:08 +0300 Subject: [PATCH 052/116] optimisations and fixes --- dataRetriever/interface.go | 1 + dataRetriever/shardedData/shardedData.go | 15 +- dataRetriever/shardedData/shardedData_test.go | 9 +- dataRetriever/txpool/interface.go | 1 + dataRetriever/txpool/shardedTxPool.go | 16 +- dataRetriever/txpool/shardedTxPool_test.go | 3 +- factory/processing/blockProcessorCreator.go | 2 + factory/processing/processComponents.go | 6 +- go.mod | 2 +- go.sum | 4 +- integrationTests/testFullNode.go | 5 +- integrationTests/testProcessorNode.go | 1 + integrationTests/testSyncNode.go | 1 + .../vm/staking/metaBlockProcessorCreator.go | 1 + process/block/argProcessor.go | 1 + process/block/baseProcess.go | 4 + process/block/baseProcess_test.go | 1 + process/block/export_test.go | 1 + process/block/metablock.go | 9 + process/block/metablock_test.go | 1 + process/block/shardblock.go | 9 + process/errors.go | 3 + process/interface.go | 20 ++ process/track/export_test.go | 10 + process/track/miniBlockTrack.go | 75 +++++++- process/track/miniBlockTrack_test.go | 181 +++++++++++++++++- testscommon/miniBlockTrackerStub.go | 26 +++ testscommon/shardedDataCacheNotifierMock.go | 4 + testscommon/shardedDataStub.go | 9 + 29 files changed, 394 insertions(+), 27 deletions(-) create mode 100644 testscommon/miniBlockTrackerStub.go diff --git a/dataRetriever/interface.go b/dataRetriever/interface.go index d4ed66847dd..a5e7045d927 100644 --- a/dataRetriever/interface.go +++ b/dataRetriever/interface.go @@ -176,6 +176,7 @@ type ShardedDataCacherNotifier interface { RemoveSetOfDataFromPool(keys [][]byte, cacheId string) ImmunizeSetOfDataAgainstEviction(keys [][]byte, cacheId string, nonce uint64) SetOldestImmuneNonce(cacheId string, nonce uint64) + SetOldestImmuneNonceForAllCaches(nonce uint64) RemoveDataFromAllShards(key []byte) MergeShardStores(sourceCacheID, destCacheID string) Clear() diff --git a/dataRetriever/shardedData/shardedData.go b/dataRetriever/shardedData/shardedData.go index dd9d4a6acdb..b3533f83375 100644 --- a/dataRetriever/shardedData/shardedData.go +++ b/dataRetriever/shardedData/shardedData.go @@ -7,11 +7,12 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/counting" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/storage/storageunit" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("dataretriever/shardeddata") @@ -200,6 +201,18 @@ func (sd *shardedData) SetOldestImmuneNonce(cacheID string, nonce uint64) { store.cache.SetOldestImmuneNonce(nonce) } +// SetOldestImmuneNonceForAllCaches deactivates immunity below the provided nonce +// on every backing shard store. Called from the shard's commit path once the +// cross-notarized metablock has advanced. +func (sd *shardedData) SetOldestImmuneNonceForAllCaches(nonce uint64) { + sd.mutShardedDataStore.RLock() + defer sd.mutShardedDataStore.RUnlock() + + for _, store := range sd.shardedDataStore { + store.cache.SetOldestImmuneNonce(nonce) + } +} + // RemoveData will remove data hash from the corresponding shard store func (sd *shardedData) RemoveData(key []byte, cacheID string) { store := sd.shardStore(cacheID) diff --git a/dataRetriever/shardedData/shardedData_test.go b/dataRetriever/shardedData/shardedData_test.go index 96434a2652f..a9764ed6b8c 100644 --- a/dataRetriever/shardedData/shardedData_test.go +++ b/dataRetriever/shardedData/shardedData_test.go @@ -10,8 +10,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/stretchr/testify/assert" + + "github.com/multiversx/mx-chain-go/storage/storageunit" ) var timeoutWaitForWaitGroups = time.Second * 2 @@ -112,7 +113,7 @@ func TestShardedData_AddDataInParallel(t *testing.T) { wg.Wait() - //checking + // checking for i := 0; i < vals; i++ { key := []byte(strconv.Itoa(i)) assert.True(t, sd.shardStore("1").cache.Has(key), fmt.Sprintf("for val %d", i)) @@ -271,10 +272,10 @@ func TestShardedData_RegisterAddedDataHandlerNotAddedShouldNotCall(t *testing.T) sd, _ := NewShardedData("", defaultTestConfig) - //first add, no call + // first add, no call sd.AddData([]byte("aaaa"), "bbbb", 4, "0") sd.RegisterOnAdded(f) - //second add, should not call as the data was found + // second add, should not call as the data was found sd.AddData([]byte("aaaa"), "bbbb", 4, "0") select { diff --git a/dataRetriever/txpool/interface.go b/dataRetriever/txpool/interface.go index 5242c5d1b57..fc446f954d3 100644 --- a/dataRetriever/txpool/interface.go +++ b/dataRetriever/txpool/interface.go @@ -4,6 +4,7 @@ import ( "math/big" "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/txcache" ) diff --git a/dataRetriever/txpool/shardedTxPool.go b/dataRetriever/txpool/shardedTxPool.go index 2759765c3c9..4c8e6a3115e 100644 --- a/dataRetriever/txpool/shardedTxPool.go +++ b/dataRetriever/txpool/shardedTxPool.go @@ -7,11 +7,12 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/counting" "github.com/multiversx/mx-chain-core-go/data" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/txcache" - logger "github.com/multiversx/mx-chain-logger-go" ) var _ dataRetriever.ShardedDataCacherNotifier = (*shardedTxPool)(nil) @@ -174,6 +175,19 @@ func (txPool *shardedTxPool) SetOldestImmuneNonce(cacheID string, nonce uint64) shard.Cache.SetOldestImmuneNonce(nonce) } +// SetOldestImmuneNonceForAllCaches deactivates immunity below the provided nonce +// on every backing cache. Called from the shard's commit path once cross-notarized +// metablock processing has advanced and the items confirmed up to (nonce - 1) +// are guaranteed to have been executed. +func (txPool *shardedTxPool) SetOldestImmuneNonceForAllCaches(nonce uint64) { + txPool.mutexBackingMap.RLock() + defer txPool.mutexBackingMap.RUnlock() + + for _, shard := range txPool.backingMap { + shard.Cache.SetOldestImmuneNonce(nonce) + } +} + // AddData adds the transaction to the cache func (txPool *shardedTxPool) AddData(key []byte, value interface{}, sizeInBytes int, cacheID string) { valueAsTransaction, ok := value.(data.TransactionHandler) diff --git a/dataRetriever/txpool/shardedTxPool_test.go b/dataRetriever/txpool/shardedTxPool_test.go index d503c269252..4b2a2e0b1d0 100644 --- a/dataRetriever/txpool/shardedTxPool_test.go +++ b/dataRetriever/txpool/shardedTxPool_test.go @@ -11,10 +11,11 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" - "github.com/stretchr/testify/require" ) func Test_NewShardedTxPool(t *testing.T) { diff --git a/factory/processing/blockProcessorCreator.go b/factory/processing/blockProcessorCreator.go index ddc6603f379..28ae0ac846e 100644 --- a/factory/processing/blockProcessorCreator.go +++ b/factory/processing/blockProcessorCreator.go @@ -432,6 +432,7 @@ func (pcf *processComponentsFactory) newShardBlockProcessor( HeaderValidator: headerValidator, BootStorer: bootStorer, BlockTracker: blockTracker, + MiniBlockTracker: pcf.miniBlockTracker, FeeHandler: txFeeHandler, BlockSizeThrottler: blockSizeThrottler, HistoryRepository: pcf.historyRepo, @@ -870,6 +871,7 @@ func (pcf *processComponentsFactory) newMetaBlockProcessor( HeaderValidator: headerValidator, BootStorer: bootStorer, BlockTracker: blockTracker, + MiniBlockTracker: pcf.miniBlockTracker, FeeHandler: txFeeHandler, BlockSizeThrottler: blockSizeThrottler, HistoryRepository: pcf.historyRepo, diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 92ac01fe17e..4263d6d4e4e 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -113,6 +113,7 @@ type processComponents struct { fullArchivePeerShardMapper process.NetworkShardingCollector apiTransactionEvaluator factory.TransactionEvaluator miniBlocksPoolCleaner process.PoolsCleaner + miniBlockTracker process.MiniBlockTracker txsPoolCleaner process.PoolsCleaner fallbackHeaderValidator process.FallbackHeaderValidator whiteListHandler process.WhiteListHandler @@ -174,6 +175,7 @@ type ProcessComponentsFactoryArgs struct { } type processComponentsFactory struct { + miniBlockTracker process.MiniBlockTracker config config.Config roundConfig config.RoundConfig epochConfig config.EpochConfig @@ -518,7 +520,7 @@ func (pcf *processComponentsFactory) Create() (*processComponents, error) { txsPoolsCleaner.StartCleaning() - _, err = track.NewMiniBlockTrack( + miniBlockTracker, err := track.NewMiniBlockTrack( pcf.data.Datapool(), blockTracker, pcf.bootstrapComponents.ShardCoordinator(), @@ -527,6 +529,7 @@ func (pcf *processComponentsFactory) Create() (*processComponents, error) { if err != nil { return nil, err } + pcf.miniBlockTracker = miniBlockTracker hardforkTrigger, err := pcf.createHardforkTrigger(epochStartTrigger) if err != nil { @@ -759,6 +762,7 @@ func (pcf *processComponentsFactory) Create() (*processComponents, error) { fullArchivePeerShardMapper: fullArchivePeerShardMapper, apiTransactionEvaluator: apiTransactionEvaluator, miniBlocksPoolCleaner: mbsPoolsCleaner, + miniBlockTracker: miniBlockTracker, txsPoolCleaner: txsPoolsCleaner, fallbackHeaderValidator: fallbackHeaderValidator, whiteListHandler: pcf.whiteListHandler, diff --git a/go.mod b/go.mod index fc8f5fd7317..a837dde8cd8 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/gogo/protobuf => github.com/multiversx/protobuf v1.3.2 -replace github.com/multiversx/mx-chain-storage-go => github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260513084840-22d627716370 +replace github.com/multiversx/mx-chain-storage-go => github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260515145423-6fa5d611f6b6 diff --git a/go.sum b/go.sum index b7cbf032404..3573ff4102d 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260513084840-22d627716370 h1:qhgeRSVZsnQqS7MBoN4XCpshKlVpXSS4fv/g49uNHJQ= -github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260513084840-22d627716370/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260515145423-6fa5d611f6b6 h1:cVkUgwm0W+egegdlRbDi4tnZB6hLmgNl9kT7uzRskSI= +github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260515145423-6fa5d611f6b6/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 4c122860f52..51f654896af 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -14,9 +14,10 @@ import ( crypto "github.com/multiversx/mx-chain-crypto-go" mclMultiSig "github.com/multiversx/mx-chain-crypto-go/signing/mcl/multisig" "github.com/multiversx/mx-chain-crypto-go/signing/multisig" - "github.com/multiversx/mx-chain-go/state/disabled" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" + "github.com/multiversx/mx-chain-go/state/disabled" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/enablers" "github.com/multiversx/mx-chain-go/common/forking" @@ -845,6 +846,7 @@ func (tpn *TestFullNode) initBlockProcessor( }, }, BlockTracker: tpn.BlockTracker, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, @@ -1086,6 +1088,7 @@ func (tpn *TestFullNode) initBlockProcessorWithSync( }, }, BlockTracker: tpn.BlockTracker, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 9151c87a2a0..c427d2a02e3 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -2284,6 +2284,7 @@ func (tpn *TestProcessorNode) initBlockProcessor() { }, }, BlockTracker: tpn.BlockTracker, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, diff --git a/integrationTests/testSyncNode.go b/integrationTests/testSyncNode.go index 41d8d5a1eba..5f1a212892d 100644 --- a/integrationTests/testSyncNode.go +++ b/integrationTests/testSyncNode.go @@ -96,6 +96,7 @@ func (tpn *TestProcessorNode) initBlockProcessorWithSync() { }, }, BlockTracker: tpn.BlockTracker, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, diff --git a/integrationTests/vm/staking/metaBlockProcessorCreator.go b/integrationTests/vm/staking/metaBlockProcessorCreator.go index bba0b69bb9e..366f21ca884 100644 --- a/integrationTests/vm/staking/metaBlockProcessorCreator.go +++ b/integrationTests/vm/staking/metaBlockProcessorCreator.go @@ -93,6 +93,7 @@ func createMetaBlockProcessor( HeaderValidator: headerValidator, BootStorer: bootStorer, BlockTracker: blockTracker, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: &mock.BlockSizeThrottlerStub{}, HistoryRepository: &dblookupext.HistoryRepositoryStub{}, VMContainersFactory: metaVMFactory, diff --git a/process/block/argProcessor.go b/process/block/argProcessor.go index e418426b0f0..6fcf6fe18b0 100644 --- a/process/block/argProcessor.go +++ b/process/block/argProcessor.go @@ -97,6 +97,7 @@ type ArgBaseProcessor struct { ManagedPeersHolder common.ManagedPeersHolder SentSignaturesTracker process.SentSignaturesTracker StateAccessesCollector state.StateAccessesCollector + MiniBlockTracker process.MiniBlockTracker } // ArgShardProcessor holds all dependencies required by the process data factory in order to create diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 5b6e315f8f9..aac0b6f965e 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -84,6 +84,7 @@ type baseProcessor struct { requestBlockBodyHandler process.RequestBlockBodyHandler requestHandler process.RequestHandler blockTracker process.BlockTracker + miniBlockTracker process.MiniBlockTracker dataPool dataRetriever.PoolsHolder feeHandler process.TransactionFeeHandler blockChain data.ChainHandler @@ -551,6 +552,9 @@ func checkProcessorParameters(arguments ArgBaseProcessor) error { if check.IfNil(arguments.BlockTracker) { return process.ErrNilBlockTracker } + if check.IfNil(arguments.MiniBlockTracker) { + return process.ErrNilMiniBlockTracker + } if check.IfNil(arguments.FeeHandler) { return process.ErrNilEconomicsFeeHandler } diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index bdbc373e89d..3d0a1d1302f 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -125,6 +125,7 @@ func createArgBaseProcessor( }, }, BlockTracker: mock.NewBlockTrackerMock(bootstrapComponents.ShardCoordinator(), startHeaders), + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: &mock.BlockSizeThrottlerStub{}, Version: "softwareVersion", HistoryRepository: &dblookupext.HistoryRepositoryStub{}, diff --git a/process/block/export_test.go b/process/block/export_test.go index d7818ece09a..227f350c0d5 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -181,6 +181,7 @@ func NewShardProcessorEmptyWith3shards( }, }, BlockTracker: mock.NewBlockTrackerMock(shardCoordinator, genesisBlocks), + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: &mock.BlockSizeThrottlerStub{}, Version: "softwareVersion", HistoryRepository: &dblookupext.HistoryRepositoryStub{}, diff --git a/process/block/metablock.go b/process/block/metablock.go index 33f0b4ac917..49a44c2d5f7 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -113,6 +113,7 @@ func NewMetaProcessor(arguments ArgMetaProcessor) (*metaProcessor, error) { roundHandler: arguments.CoreComponents.RoundHandler(), bootStorer: arguments.BootStorer, blockTracker: arguments.BlockTracker, + miniBlockTracker: arguments.MiniBlockTracker, dataPool: arguments.DataComponents.Datapool(), blockChain: arguments.DataComponents.Blockchain(), outportHandler: arguments.StatusComponents.OutportHandler(), @@ -1845,6 +1846,14 @@ func (mp *metaProcessor) saveLastNotarizedHeader(header *block.MetaBlock) error hash := lastCrossNotarizedHeaderForShard[shardID].hash mp.blockTracker.AddCrossNotarizedHeader(shardID, hdr, hash) DisplayLastNotarized(mp.marshalizer, mp.hasher, hdr, shardID) + + // Per-shard threshold advance: commitAll already ran, so SCRs from shardID up to + // hdr.GetNonce() can be released. hdr.GetNonce()+1 releases items at the just- + // notarized nonce too, since they were processed in this metablock. + if !check.IfNil(hdr) && !check.IfNil(mp.miniBlockTracker) { + threshold := hdr.GetNonce() + 1 + mp.miniBlockTracker.ReleaseImmunityForCommittedShardBlocks(shardID, threshold) + } } return nil diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index ee2e2c5fd60..0e4d40756ab 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -147,6 +147,7 @@ func createMockMetaArguments( }, }, BlockTracker: mock.NewBlockTrackerMock(bootstrapComponents.ShardCoordinator(), startHeaders), + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: &mock.BlockSizeThrottlerStub{}, HistoryRepository: &dblookupext.HistoryRepositoryStub{}, ScheduledTxsExecutionHandler: &testscommon.ScheduledTxsExecutionStub{}, diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 8b93dc145f2..2deffac6349 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -98,6 +98,7 @@ func NewShardProcessor(arguments ArgShardProcessor) (*shardProcessor, error) { headerValidator: arguments.HeaderValidator, bootStorer: arguments.BootStorer, blockTracker: arguments.BlockTracker, + miniBlockTracker: arguments.MiniBlockTracker, dataPool: arguments.DataComponents.Datapool(), blockChain: arguments.DataComponents.Blockchain(), feeHandler: arguments.FeeHandler, @@ -1518,6 +1519,14 @@ func (sp *shardProcessor) saveLastNotarizedHeader(shardId uint32, processedHdrs sp.blockTracker.AddCrossNotarizedHeader(shardId, lastCrossNotarizedHeader, lastCrossNotarizedHeaderHash) DisplayLastNotarized(sp.marshalizer, sp.hasher, lastCrossNotarizedHeader, shardId) + // processedHdrs only contains fully-processed metablocks (see processedAll gate in + // getOrderedProcessedMetaBlocksFromMiniBlockHashes), so lastNonce+1 releases items + // from those metablocks now that the consuming shard block is being committed. + if shardId == core.MetachainShardId && !check.IfNil(lastCrossNotarizedHeader) && !check.IfNil(sp.miniBlockTracker) { + threshold := lastCrossNotarizedHeader.GetNonce() + 1 + sp.miniBlockTracker.ReleaseImmunityForCommittedMetaBlocks(threshold) + } + return nil } diff --git a/process/errors.go b/process/errors.go index dabdef5f176..5f0f6385e91 100644 --- a/process/errors.go +++ b/process/errors.go @@ -594,6 +594,9 @@ var ErrNilPeerShardMapper = errors.New("nil peer shard mapper") // ErrNilBlockTracker signals that a nil block tracker was provided var ErrNilBlockTracker = errors.New("nil block tracker") +// ErrNilMiniBlockTracker signals that a nil miniblock tracker was provided +var ErrNilMiniBlockTracker = errors.New("nil miniblock tracker") + // ErrHeaderIsBlackListed signals that the header provided is black listed var ErrHeaderIsBlackListed = errors.New("header is black listed") diff --git a/process/interface.go b/process/interface.go index 99bafaa1354..a0b48a7f53b 100644 --- a/process/interface.go +++ b/process/interface.go @@ -922,6 +922,26 @@ type BlockTracker interface { IsInterfaceNil() bool } +// MiniBlockTracker tracks the confirmation status of cross-shard miniblocks so that +// their referenced transactions can be granted immunity in the pool on metablock +// arrival and released from immunity on this shard's commit. +type MiniBlockTracker interface { + // ReleaseImmunityForCommittedMetaBlocks is called by the shard processor after + // metablocks up to (threshold-1) have been fully processed. It advances the + // immunity threshold for every cache on every pool and drops stale registry + // entries whose tracked nonce is strictly below `threshold`. + ReleaseImmunityForCommittedMetaBlocks(threshold uint64) + + // ReleaseImmunityForCommittedShardBlocks is called by the meta processor after + // shard headers from `senderShard` up to (threshold-1) have been fully processed. + // It advances the immunity threshold for caches whose senderShardID matches + // `senderShard` and receiver is the metachain, and drops the corresponding stale + // registry entries. + ReleaseImmunityForCommittedShardBlocks(senderShard uint32, threshold uint64) + + IsInterfaceNil() bool +} + // FloodPreventer defines the behavior of a component that is able to signal that too many events occurred // on a provided identifier between Reset calls type FloodPreventer interface { diff --git a/process/track/export_test.go b/process/track/export_test.go index 8cbcccb2919..9a11f9ad0f2 100644 --- a/process/track/export_test.go +++ b/process/track/export_test.go @@ -4,6 +4,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" @@ -286,3 +287,12 @@ func (mbt *miniBlockTrack) GetTransactionPool(mbType block.Type) dataRetriever.S func (mbt *miniBlockTrack) SetBlockTransactionsPool(blockTransactionsPool dataRetriever.ShardedDataCacherNotifier) { mbt.blockTransactionsPool = blockTransactionsPool } + +// GetConfirmedMiniBlockInfo - test accessor for the local registry +func (mbt *miniBlockTrack) GetConfirmedMiniBlockInfo(miniBlockHash []byte) (cacheID string, nonce uint64, ok bool) { + info, found := mbt.getConfirmedMiniBlockInfo(miniBlockHash) + if !found { + return "", 0, false + } + return info.cacheID, info.nonce, true +} diff --git a/process/track/miniBlockTrack.go b/process/track/miniBlockTrack.go index 27aaa52ab27..8b8225d0283 100644 --- a/process/track/miniBlockTrack.go +++ b/process/track/miniBlockTrack.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" @@ -165,14 +166,11 @@ func (mbt *miniBlockTrack) registerFromMiniBlockHeaders( selfShardID := mbt.shardCoordinator.SelfId() for _, miniBlockHeader := range miniBlockHeaders { receiverShard := miniBlockHeader.GetReceiverShardID() + // AllShardId from a metaheader (e.g. rewards) is treated as receiver = self. receiverIsAllShardsMiniBlockFromMetaHeader := receiverShard == core.AllShardId && processingShard == core.MetachainShardId receiverIsRelevantForCurrentShard := receiverShard == selfShardID || receiverIsAllShardsMiniBlockFromMetaHeader senderShard := miniBlockHeader.GetSenderShardID() senderIsSelfShard := senderShard == selfShardID - // Track only miniblocks that are relevant for this shard and come from another shard. - // This includes direct cross-shard miniblocks addressed to this shard and the - // special metachain-header case where the receiver is AllShardId. - // Intra-shard miniblocks are produced and processed locally, so they are skipped here. if !receiverIsRelevantForCurrentShard || senderIsSelfShard { continue } @@ -189,9 +187,9 @@ func (mbt *miniBlockTrack) registerFromMiniBlockHeaders( continue } + // Threshold advance is deferred to commit (see ReleaseImmunityForCommittedMetaBlocks). + // Advancing here would release items from older metablocks before this shard executes them. mbt.storeConfirmedMiniBlockInfo(miniBlockHeader.GetHash(), mbInfo) - transactionPool.SetOldestImmuneNonce(cacheID, nonce) - mbt.cleanupConfirmedMiniBlocks(cacheID, nonce) mbt.tryProcessStoredMiniBlock(miniBlockHeader.GetHash(), mbInfo) } } @@ -218,7 +216,6 @@ func (mbt *miniBlockTrack) immunizeMiniBlock(miniBlockHash []byte, miniBlock *bl } mbt.whitelistHandler.Add(miniBlock.TxHashes) - transactionPool.SetOldestImmuneNonce(confirmationInfo.cacheID, confirmationInfo.nonce) transactionPool.ImmunizeSetOfDataAgainstEviction(miniBlock.TxHashes, confirmationInfo.cacheID, confirmationInfo.nonce) mbt.removeConfirmedMiniBlockInfo(miniBlockHash) } @@ -250,15 +247,75 @@ func (mbt *miniBlockTrack) removeConfirmedMiniBlockInfo(miniBlockHash []byte) { mbt.mutConfirmedMiniBlocks.Unlock() } -func (mbt *miniBlockTrack) cleanupConfirmedMiniBlocks(cacheID string, nonce uint64) { +// CleanupConfirmedMiniBlocksBelow drops every tracked confirmation whose nonce +// is strictly below `threshold`. Called from the shard's commit path alongside +// SetOldestImmuneNonceForAllCaches so that the local registry doesn't accumulate +// stale entries for miniblocks that never arrived in the pool. +func (mbt *miniBlockTrack) CleanupConfirmedMiniBlocksBelow(threshold uint64) { + mbt.mutConfirmedMiniBlocks.Lock() + defer mbt.mutConfirmedMiniBlocks.Unlock() + + for key, info := range mbt.confirmedMiniBlocks { + if info.nonce >= threshold { + continue + } + + delete(mbt.confirmedMiniBlocks, key) + } +} + +// CleanupConfirmedMiniBlocksBelowForCacheID drops every tracked confirmation whose +// cacheID matches and nonce is strictly below `threshold`. Used by the meta commit +// path where the threshold is per-sender-shard rather than uniform. +func (mbt *miniBlockTrack) CleanupConfirmedMiniBlocksBelowForCacheID(cacheID string, threshold uint64) { mbt.mutConfirmedMiniBlocks.Lock() defer mbt.mutConfirmedMiniBlocks.Unlock() for key, info := range mbt.confirmedMiniBlocks { - if info.cacheID != cacheID || info.nonce >= nonce { + if info.cacheID != cacheID || info.nonce >= threshold { continue } delete(mbt.confirmedMiniBlocks, key) } } + +// ReleaseImmunityForCommittedMetaBlocks advances the immunity threshold uniformly +// across every tx-pool cache and prunes the local registry for entries below +// `threshold`. Called from the shard's commit path once the cross-notarized +// metablock has advanced past (threshold-1). +func (mbt *miniBlockTrack) ReleaseImmunityForCommittedMetaBlocks(threshold uint64) { + if !check.IfNil(mbt.blockTransactionsPool) { + mbt.blockTransactionsPool.SetOldestImmuneNonceForAllCaches(threshold) + } + if !check.IfNil(mbt.rewardTransactionsPool) { + mbt.rewardTransactionsPool.SetOldestImmuneNonceForAllCaches(threshold) + } + if !check.IfNil(mbt.unsignedTransactionsPool) { + mbt.unsignedTransactionsPool.SetOldestImmuneNonceForAllCaches(threshold) + } + mbt.CleanupConfirmedMiniBlocksBelow(threshold) +} + +// ReleaseImmunityForCommittedShardBlocks advances the immunity threshold only on +// caches with senderShardID = `senderShard` and receiver = metachain, and prunes +// the local registry for matching entries below `threshold`. Called from the +// meta processor after its cross-notarized shard header has advanced for `senderShard`. +func (mbt *miniBlockTrack) ReleaseImmunityForCommittedShardBlocks(senderShard uint32, threshold uint64) { + cacheID := process.ShardCacherIdentifier(senderShard, core.MetachainShardId) + if !check.IfNil(mbt.blockTransactionsPool) { + mbt.blockTransactionsPool.SetOldestImmuneNonce(cacheID, threshold) + } + if !check.IfNil(mbt.rewardTransactionsPool) { + mbt.rewardTransactionsPool.SetOldestImmuneNonce(cacheID, threshold) + } + if !check.IfNil(mbt.unsignedTransactionsPool) { + mbt.unsignedTransactionsPool.SetOldestImmuneNonce(cacheID, threshold) + } + mbt.CleanupConfirmedMiniBlocksBelowForCacheID(cacheID, threshold) +} + +// IsInterfaceNil returns true if the receiver is a nil interface +func (mbt *miniBlockTrack) IsInterfaceNil() bool { + return mbt == nil +} diff --git a/process/track/miniBlockTrack_test.go b/process/track/miniBlockTrack_test.go index 4fc384d3239..89861bb89cb 100644 --- a/process/track/miniBlockTrack_test.go +++ b/process/track/miniBlockTrack_test.go @@ -420,14 +420,12 @@ func TestRegisterConfirmedMiniBlocksForHeader_ShouldImmunizeStoredMiniBlock(t *t }, } var immunizedKeys [][]byte - var setOldestImmuneNonceCacheID string var immunizedCacheID string - var setOldestImmuneNonceNonce uint64 var immunizedNonce uint64 + setOldestImmuneNonceCalled := false blockTransactionsPool := &testscommon.ShardedDataStub{ SetOldestImmuneNonceCalled: func(cacheID string, nonce uint64) { - setOldestImmuneNonceCacheID = cacheID - setOldestImmuneNonceNonce = nonce + setOldestImmuneNonceCalled = true }, ImmunizeSetOfDataAgainstEvictionCalled: func(keys [][]byte, destCacheID string, nonce uint64) { immunizedKeys = keys @@ -458,10 +456,181 @@ func TestRegisterConfirmedMiniBlocksForHeader_ShouldImmunizeStoredMiniBlock(t *t }, }, nil) + // Immunization happens on metablock arrival. assert.Equal(t, txHashes, whitelistedKeys) assert.Equal(t, txHashes, immunizedKeys) - assert.Equal(t, process.ShardCacherIdentifier(1, 0), setOldestImmuneNonceCacheID) assert.Equal(t, process.ShardCacherIdentifier(1, 0), immunizedCacheID) - assert.Equal(t, uint64(7), setOldestImmuneNonceNonce) assert.Equal(t, uint64(7), immunizedNonce) + // Regression guard: threshold advance is deferred to commit. + assert.False(t, setOldestImmuneNonceCalled, "SetOldestImmuneNonce must not be called from metablock arrival path") +} + +func TestMiniBlockTrack_CleanupConfirmedMiniBlocksBelow(t *testing.T) { + t.Parallel() + + dataPool := createDataPool() + blockTracker := &mock.BlockTrackerMock{} + var finalMetachainHeadersHandler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte) + blockTracker.RegisterFinalMetachainHeadersHandlerCalled = func(handler func(shardID uint32, headers []data.HeaderHandler, headersHashes [][]byte)) { + finalMetachainHeadersHandler = handler + } + + mbt, _ := track.NewMiniBlockTrack(dataPool, blockTracker, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + + // Two confirmed miniblocks at different nonces, neither arriving in pool. + finalMetachainHeadersHandler(core.MetachainShardId, []data.HeaderHandler{ + &block.MetaBlock{ + Nonce: 5, + ShardInfo: []block.ShardData{ + { + ShardID: 1, + ShardMiniBlockHeaders: []block.MiniBlockHeader{ + {Hash: []byte("mb_old"), SenderShardID: 1, ReceiverShardID: 0, Type: block.TxBlock}, + }, + }, + }, + }, + }, nil) + finalMetachainHeadersHandler(core.MetachainShardId, []data.HeaderHandler{ + &block.MetaBlock{ + Nonce: 10, + ShardInfo: []block.ShardData{ + { + ShardID: 1, + ShardMiniBlockHeaders: []block.MiniBlockHeader{ + {Hash: []byte("mb_new"), SenderShardID: 1, ReceiverShardID: 0, Type: block.TxBlock}, + }, + }, + }, + }, + }, nil) + + // Cleanup with threshold 8 should drop the nonce-5 entry but keep nonce-10. + mbt.CleanupConfirmedMiniBlocksBelow(8) + + _, _, hasOld := mbt.GetConfirmedMiniBlockInfo([]byte("mb_old")) + _, _, hasNew := mbt.GetConfirmedMiniBlockInfo([]byte("mb_new")) + assert.False(t, hasOld) + assert.True(t, hasNew) +} + +func TestMiniBlockTrack_ReleaseImmunityForCommittedMetaBlocks(t *testing.T) { + t.Parallel() + + miniBlocksPool := cache.NewCacherStub() + miniBlocksPool.PeekCalled = func(_ []byte) (interface{}, bool) { return nil, false } + + var blockPoolThreshold, rewardPoolThreshold, unsignedPoolThreshold uint64 + blockPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceForAllCachesCalled: func(nonce uint64) { blockPoolThreshold = nonce }, + } + rewardPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceForAllCachesCalled: func(nonce uint64) { rewardPoolThreshold = nonce }, + } + unsignedPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceForAllCachesCalled: func(nonce uint64) { unsignedPoolThreshold = nonce }, + } + + dataPool := &dataRetrieverMock.PoolsHolderStub{ + TransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return blockPool }, + RewardTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return rewardPool }, + UnsignedTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return unsignedPool }, + MiniBlocksCalled: func() storage.Cacher { return miniBlocksPool }, + } + + blockTracker := &mock.BlockTrackerMock{} + var headersHandler func(uint32, []data.HeaderHandler, [][]byte) + blockTracker.RegisterFinalMetachainHeadersHandlerCalled = func(handler func(uint32, []data.HeaderHandler, [][]byte)) { + headersHandler = handler + } + + mbt, _ := track.NewMiniBlockTrack(dataPool, blockTracker, mock.NewMultipleShardsCoordinatorMock(), &testscommon.WhiteListHandlerStub{}) + + // Seed registry with entries at two different nonces. + headersHandler(core.MetachainShardId, []data.HeaderHandler{ + &block.MetaBlock{Nonce: 5, ShardInfo: []block.ShardData{{ShardID: 1, ShardMiniBlockHeaders: []block.MiniBlockHeader{ + {Hash: []byte("mb_old"), SenderShardID: 1, ReceiverShardID: 0, Type: block.TxBlock}, + }}}}, + }, nil) + headersHandler(core.MetachainShardId, []data.HeaderHandler{ + &block.MetaBlock{Nonce: 10, ShardInfo: []block.ShardData{{ShardID: 1, ShardMiniBlockHeaders: []block.MiniBlockHeader{ + {Hash: []byte("mb_new"), SenderShardID: 1, ReceiverShardID: 0, Type: block.TxBlock}, + }}}}, + }, nil) + + mbt.ReleaseImmunityForCommittedMetaBlocks(8) + + // All three pools should have received the threshold uniformly. + assert.Equal(t, uint64(8), blockPoolThreshold) + assert.Equal(t, uint64(8), rewardPoolThreshold) + assert.Equal(t, uint64(8), unsignedPoolThreshold) + + // Registry pruned below threshold. + _, _, hasOld := mbt.GetConfirmedMiniBlockInfo([]byte("mb_old")) + _, _, hasNew := mbt.GetConfirmedMiniBlockInfo([]byte("mb_new")) + assert.False(t, hasOld) + assert.True(t, hasNew) +} + +func TestMiniBlockTrack_ReleaseImmunityForCommittedShardBlocks(t *testing.T) { + t.Parallel() + + miniBlocksPool := cache.NewCacherStub() + miniBlocksPool.PeekCalled = func(_ []byte) (interface{}, bool) { return nil, false } + + type call struct { + cacheID string + nonce uint64 + } + var blockCalls, rewardCalls, unsignedCalls []call + blockPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceCalled: func(c string, n uint64) { blockCalls = append(blockCalls, call{c, n}) }, + } + rewardPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceCalled: func(c string, n uint64) { rewardCalls = append(rewardCalls, call{c, n}) }, + } + unsignedPool := &testscommon.ShardedDataStub{ + SetOldestImmuneNonceCalled: func(c string, n uint64) { unsignedCalls = append(unsignedCalls, call{c, n}) }, + } + + dataPool := &dataRetrieverMock.PoolsHolderStub{ + TransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return blockPool }, + RewardTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return rewardPool }, + UnsignedTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return unsignedPool }, + MiniBlocksCalled: func() storage.Cacher { return miniBlocksPool }, + } + + shardCoordinator := mock.NewMultipleShardsCoordinatorMock() + shardCoordinator.CurrentShard = core.MetachainShardId + + blockTracker := &mock.BlockTrackerMock{} + var crossHeadersHandler func(uint32, []data.HeaderHandler, [][]byte) + blockTracker.RegisterCrossNotarizedHeadersHandlerCalled = func(handler func(uint32, []data.HeaderHandler, [][]byte)) { + crossHeadersHandler = handler + } + + mbt, _ := track.NewMiniBlockTrack(dataPool, blockTracker, shardCoordinator, &testscommon.WhiteListHandlerStub{}) + + // Seed registry: SCR from shard 1 to meta at nonce 5, and unrelated entry from shard 2 to meta at nonce 5. + crossHeadersHandler(0, []data.HeaderHandler{ + &block.Header{Nonce: 5, ShardID: 1, MiniBlockHeaders: []block.MiniBlockHeader{ + {Hash: []byte("scr_shard1"), SenderShardID: 1, ReceiverShardID: core.MetachainShardId, Type: block.SmartContractResultBlock}, + }}, + &block.Header{Nonce: 5, ShardID: 2, MiniBlockHeaders: []block.MiniBlockHeader{ + {Hash: []byte("scr_shard2"), SenderShardID: 2, ReceiverShardID: core.MetachainShardId, Type: block.SmartContractResultBlock}, + }}, + }, nil) + + mbt.ReleaseImmunityForCommittedShardBlocks(1, 6) + + expectedCacheID := process.ShardCacherIdentifier(1, core.MetachainShardId) + assert.Equal(t, []call{{expectedCacheID, 6}}, blockCalls) + assert.Equal(t, []call{{expectedCacheID, 6}}, rewardCalls) + assert.Equal(t, []call{{expectedCacheID, 6}}, unsignedCalls) + + // Only the shard-1 registry entry should be pruned. + _, _, hasShard1 := mbt.GetConfirmedMiniBlockInfo([]byte("scr_shard1")) + _, _, hasShard2 := mbt.GetConfirmedMiniBlockInfo([]byte("scr_shard2")) + assert.False(t, hasShard1) + assert.True(t, hasShard2) } diff --git a/testscommon/miniBlockTrackerStub.go b/testscommon/miniBlockTrackerStub.go new file mode 100644 index 00000000000..6d30d72d01c --- /dev/null +++ b/testscommon/miniBlockTrackerStub.go @@ -0,0 +1,26 @@ +package testscommon + +// MiniBlockTrackerStub is a stub for process.MiniBlockTracker +type MiniBlockTrackerStub struct { + ReleaseImmunityForCommittedMetaBlocksCalled func(threshold uint64) + ReleaseImmunityForCommittedShardBlocksCalled func(senderShard uint32, threshold uint64) +} + +// ReleaseImmunityForCommittedMetaBlocks - +func (s *MiniBlockTrackerStub) ReleaseImmunityForCommittedMetaBlocks(threshold uint64) { + if s.ReleaseImmunityForCommittedMetaBlocksCalled != nil { + s.ReleaseImmunityForCommittedMetaBlocksCalled(threshold) + } +} + +// ReleaseImmunityForCommittedShardBlocks - +func (s *MiniBlockTrackerStub) ReleaseImmunityForCommittedShardBlocks(senderShard uint32, threshold uint64) { + if s.ReleaseImmunityForCommittedShardBlocksCalled != nil { + s.ReleaseImmunityForCommittedShardBlocksCalled(senderShard, threshold) + } +} + +// IsInterfaceNil returns true if the receiver is nil +func (s *MiniBlockTrackerStub) IsInterfaceNil() bool { + return s == nil +} diff --git a/testscommon/shardedDataCacheNotifierMock.go b/testscommon/shardedDataCacheNotifierMock.go index f8a341c5b8b..0d89985ace1 100644 --- a/testscommon/shardedDataCacheNotifierMock.go +++ b/testscommon/shardedDataCacheNotifierMock.go @@ -83,6 +83,10 @@ func (mock *ShardedDataCacheNotifierMock) ImmunizeSetOfDataAgainstEviction(_ [][ func (mock *ShardedDataCacheNotifierMock) SetOldestImmuneNonce(_ string, _ uint64) { } +// SetOldestImmuneNonceForAllCaches - +func (mock *ShardedDataCacheNotifierMock) SetOldestImmuneNonceForAllCaches(_ uint64) { +} + // RemoveDataFromAllShards - func (mock *ShardedDataCacheNotifierMock) RemoveDataFromAllShards(key []byte) { mock.mutCaches.RLock() diff --git a/testscommon/shardedDataStub.go b/testscommon/shardedDataStub.go index cba3aa8e40a..63b2c310eba 100644 --- a/testscommon/shardedDataStub.go +++ b/testscommon/shardedDataStub.go @@ -2,6 +2,7 @@ package testscommon import ( "github.com/multiversx/mx-chain-core-go/core/counting" + "github.com/multiversx/mx-chain-go/storage" ) @@ -20,6 +21,7 @@ type ShardedDataStub struct { RemoveSetOfDataFromPoolCalled func(keys [][]byte, destCacheID string) ImmunizeSetOfDataAgainstEvictionCalled func(keys [][]byte, cacheID string, nonce uint64) SetOldestImmuneNonceCalled func(cacheID string, nonce uint64) + SetOldestImmuneNonceForAllCachesCalled func(nonce uint64) CreateShardStoreCalled func(destCacheID string) GetCountsCalled func() counting.CountsWithSize KeysCalled func() [][]byte @@ -116,6 +118,13 @@ func (sd *ShardedDataStub) SetOldestImmuneNonce(cacheID string, nonce uint64) { } } +// SetOldestImmuneNonceForAllCaches - +func (sd *ShardedDataStub) SetOldestImmuneNonceForAllCaches(nonce uint64) { + if sd.SetOldestImmuneNonceForAllCachesCalled != nil { + sd.SetOldestImmuneNonceForAllCachesCalled(nonce) + } +} + // GetCounts - func (sd *ShardedDataStub) GetCounts() counting.CountsWithSize { if sd.GetCountsCalled != nil { From 0851f3d50d9579ad37365f87d4ac632c26f88a38 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 18 May 2026 12:06:52 +0300 Subject: [PATCH 053/116] peer mb check --- process/block/metablock.go | 4 ++++ process/block/metablock_test.go | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/process/block/metablock.go b/process/block/metablock.go index 872a799d2ae..0c3303d01a3 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -567,6 +567,10 @@ func (mp *metaProcessor) verifyNonEpochStartMiniBlocks(metaBlock *block.MetaBloc if miniBlockHeader.GetType() == block.RewardsBlock { return process.ErrInvalidMiniBlockType } + + if miniBlockHeader.GetType() == block.PeerBlock { + return process.ErrInvalidMiniBlockType + } } return nil diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index a187aa5f2db..d8b24e5789d 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -968,6 +968,46 @@ func TestMetaProcessor_ProcessBlock_MiniBlockChecks(t *testing.T) { require.Equal(t, process.ErrInvalidMiniBlockType, err) }) + t.Run("non epoch start should not have peer mb", func(t *testing.T) { + mb1 := &block.MiniBlock{ + TxHashes: [][]byte{[]byte("txHash1")}, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.PeerBlock, + } + + mbHash, _ := core.CalculateHash(coreComponents.IntMarsh, coreComponents.Hash, mb1) + + metaBlock := &block.MetaBlock{ + Nonce: 1, + Round: 1, + PrevHash: hash, + AccumulatedFees: big.NewInt(0), + AccumulatedFeesInEpoch: big.NewInt(0), + DeveloperFees: big.NewInt(0), + DevFeesInEpoch: big.NewInt(0), + TxCount: 1, + MiniBlockHeaders: []block.MiniBlockHeader{ + { + Hash: mbHash, + SenderShardID: core.MetachainShardId, + ReceiverShardID: 1, + Type: block.PeerBlock, + TxCount: 1, + }, + }, + } + + body := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + mb1, + }, + } + + err := mp.ProcessBlock(metaBlock, body, func() time.Duration { return time.Second }) + require.Equal(t, process.ErrInvalidMiniBlockType, err) + }) + t.Run("epoch start should have rewards or peer mb", func(t *testing.T) { mb1 := &block.MiniBlock{ TxHashes: [][]byte{[]byte("txHash1")}, From 7aafb46042175cec438d8c3455bc25b312302a92 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 18 May 2026 12:33:59 +0300 Subject: [PATCH 054/116] peer mb check - shard --- process/block/baseProcess.go | 11 +++++++++++ process/block/metablock.go | 14 -------------- process/block/shardblock.go | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index d02c91c4965..55a52b59313 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -2426,3 +2426,14 @@ func (bp *baseProcessor) checkReceivedProofIfAttestingIsNeeded(proof data.Header bp.chRcvAllHdrs <- true } } + +func (bp *baseProcessor) verifyNonEpochStartMiniBlocks(header data.HeaderHandler) error { + for _, miniBlockHeader := range header.GetMiniBlockHeaderHandlers() { + if miniBlockHeader.GetTypeInt32() == int32(block.RewardsBlock) || + miniBlockHeader.GetTypeInt32() == int32(block.PeerBlock) { + return process.ErrInvalidMiniBlockType + } + } + + return nil +} diff --git a/process/block/metablock.go b/process/block/metablock.go index 0c3303d01a3..33d09c1cec1 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -562,20 +562,6 @@ func (mp *metaProcessor) verifyEpochStartMiniBlocks(metaBlock *block.MetaBlock) return nil } -func (mp *metaProcessor) verifyNonEpochStartMiniBlocks(metaBlock *block.MetaBlock) error { - for _, miniBlockHeader := range metaBlock.MiniBlockHeaders { - if miniBlockHeader.GetType() == block.RewardsBlock { - return process.ErrInvalidMiniBlockType - } - - if miniBlockHeader.GetType() == block.PeerBlock { - return process.ErrInvalidMiniBlockType - } - } - - return nil -} - // SetNumProcessedObj will set the num of processed headers func (mp *metaProcessor) SetNumProcessedObj(numObj uint64) { mp.headersCounter.shardMBHeadersTotalProcessed = numObj diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 10485618b72..71df9c67b46 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -230,6 +230,13 @@ func (sp *shardProcessor) ProcessBlock( return err } + if !header.IsStartOfEpochBlock() { + err = sp.verifyNonEpochStartMiniBlocks(header) + if err != nil { + return err + } + } + txCounts, rewardCounts, unsignedCounts := sp.txCounter.getPoolCounts(sp.dataPool) log.Debug("total txs in pool", "counts", txCounts.String()) log.Debug("total txs in rewards pool", "counts", rewardCounts.String()) @@ -387,6 +394,17 @@ func (sp *shardProcessor) ProcessBlock( return nil } +func (sp *shardProcessor) verifyNonEpochStartMiniBlocks(header data.HeaderHandler) error { + for _, miniBlockHeader := range header.GetMiniBlockHeaderHandlers() { + if miniBlockHeader.GetTypeInt32() == int32(block.RewardsBlock) || + miniBlockHeader.GetTypeInt32() == int32(block.PeerBlock) { + return process.ErrInvalidMiniBlockType + } + } + + return nil +} + func (sp *shardProcessor) requestEpochStartInfo(header data.ShardHeaderHandler, haveTime func() time.Duration) error { if !header.IsStartOfEpochBlock() { return nil From 22e081d718427bdc70f901bcd6f2d6e7a64c9267 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 18 May 2026 13:24:12 +0300 Subject: [PATCH 055/116] remove shard check --- process/block/shardblock.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 71df9c67b46..ddfd9bd524d 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -230,13 +230,6 @@ func (sp *shardProcessor) ProcessBlock( return err } - if !header.IsStartOfEpochBlock() { - err = sp.verifyNonEpochStartMiniBlocks(header) - if err != nil { - return err - } - } - txCounts, rewardCounts, unsignedCounts := sp.txCounter.getPoolCounts(sp.dataPool) log.Debug("total txs in pool", "counts", txCounts.String()) log.Debug("total txs in rewards pool", "counts", rewardCounts.String()) From 1c838c52db42ccbdfb61d19f8c3b023576a55208 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 18 May 2026 13:26:27 +0300 Subject: [PATCH 056/116] remove shard check --- process/block/shardblock.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/process/block/shardblock.go b/process/block/shardblock.go index ddfd9bd524d..10485618b72 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -387,17 +387,6 @@ func (sp *shardProcessor) ProcessBlock( return nil } -func (sp *shardProcessor) verifyNonEpochStartMiniBlocks(header data.HeaderHandler) error { - for _, miniBlockHeader := range header.GetMiniBlockHeaderHandlers() { - if miniBlockHeader.GetTypeInt32() == int32(block.RewardsBlock) || - miniBlockHeader.GetTypeInt32() == int32(block.PeerBlock) { - return process.ErrInvalidMiniBlockType - } - } - - return nil -} - func (sp *shardProcessor) requestEpochStartInfo(header data.ShardHeaderHandler, haveTime func() time.Duration) error { if !header.IsStartOfEpochBlock() { return nil From 3b18432d6fbed6fffbf4e709eff7f3e3fdb5b607 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 18 May 2026 15:22:04 +0300 Subject: [PATCH 057/116] add tests --- dataRetriever/shardedData/shardedData_test.go | 31 +++++++ dataRetriever/txpool/shardedTxPool_test.go | 29 +++++++ process/block/metablock_test.go | 42 ++++++++++ process/block/shardblock_test.go | 80 +++++++++++++++++++ testscommon/cache/immunityCacheSpy.go | 21 +++++ 5 files changed, 203 insertions(+) create mode 100644 testscommon/cache/immunityCacheSpy.go diff --git a/dataRetriever/shardedData/shardedData_test.go b/dataRetriever/shardedData/shardedData_test.go index a9764ed6b8c..157754d4f53 100644 --- a/dataRetriever/shardedData/shardedData_test.go +++ b/dataRetriever/shardedData/shardedData_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/multiversx/mx-chain-go/storage/storageunit" + cacheStubs "github.com/multiversx/mx-chain-go/testscommon/cache" ) var timeoutWaitForWaitGroups = time.Second * 2 @@ -335,6 +336,36 @@ func TestShardedData_ImmunizeSetOfDataAgainstEviction(t *testing.T) { sd.SetOldestImmuneNonce("0", 7) } +func TestShardedData_SetOldestImmuneNonceForAllCaches(t *testing.T) { + t.Parallel() + + sd, _ := NewShardedData("", defaultTestConfig) + + cacheIDs := []string{"0", "1", "2_0"} + received := make(map[string]uint64) + var mu sync.Mutex + for _, id := range cacheIDs { + idCopy := id + spy := &cacheStubs.ImmunityCacheSpy{ + CacherStub: cacheStubs.NewCacherStub(), + SetOldestImmuneNonceCalled: func(nonce uint64) { + mu.Lock() + received[idCopy] = nonce + mu.Unlock() + }, + } + sd.shardedDataStore[id] = &shardStore{cacheID: id, cache: spy} + } + + sd.SetOldestImmuneNonceForAllCaches(42) + + assert.Equal(t, len(cacheIDs), len(sd.shardedDataStore), "no new stores should be created") + assert.Equal(t, len(cacheIDs), len(received), "every store should receive the threshold once") + for _, id := range cacheIDs { + assert.Equal(t, uint64(42), received[id], "store %s did not receive the threshold", id) + } +} + func TestShardedData_GetCounts(t *testing.T) { t.Parallel() diff --git a/dataRetriever/txpool/shardedTxPool_test.go b/dataRetriever/txpool/shardedTxPool_test.go index 4b2a2e0b1d0..04b766e308e 100644 --- a/dataRetriever/txpool/shardedTxPool_test.go +++ b/dataRetriever/txpool/shardedTxPool_test.go @@ -363,6 +363,35 @@ func TestShardedTxPool_ImmunizeSetOfDataAgainstEviction(t *testing.T) { pool.SetOldestImmuneNonce("0", 7) } +func TestShardedTxPool_SetOldestImmuneNonceForAllCaches(t *testing.T) { + t.Parallel() + + poolAsInterface, _ := newTxPoolToTest() + pool := poolAsInterface.(*shardedTxPool) + + cacheIDs := []string{"0", "1_0", "2_0"} + received := make(map[string]uint64) + var mu sync.Mutex + for _, id := range cacheIDs { + idCopy := id + mock := txcachemocks.NewTxCacheStub() + mock.SetOldestImmuneNonceCalled = func(nonce uint64) { + mu.Lock() + received[idCopy] = nonce + mu.Unlock() + } + pool.backingMap[id] = &txPoolShard{CacheID: id, Cache: mock} + } + + pool.SetOldestImmuneNonceForAllCaches(42) + + require.Equal(t, len(cacheIDs), len(pool.backingMap), "no new caches should be created") + require.Equal(t, len(cacheIDs), len(received), "every cache should receive the threshold once") + for _, id := range cacheIDs { + require.Equal(t, uint64(42), received[id], "cache %s did not receive the threshold", id) + } +} + func Test_IsInterfaceNil(t *testing.T) { poolAsInterface, _ := newTxPoolToTest() require.False(t, check.IfNil(poolAsInterface)) diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index 0e4d40756ab..21bc1a8ca6e 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -1963,6 +1963,48 @@ func TestMetaProcessor_CreateLastNotarizedHdrs(t *testing.T) { assert.Equal(t, currHdr, mp.LastNotarizedHdrForShard(currHdr.ShardID)) } +func TestMetaProcessor_SaveLastNotarizedHeader_ReleasesImmunityForCommittedShardBlocks(t *testing.T) { + t.Parallel() + + pool := dataRetrieverMock.NewPoolsHolderMock() + noOfShards := uint32(3) + coreComponents, dataComponents, bootstrapComponents, statusComponents := createMockComponentHolders() + coreComponents.Hash = &hashingMocks.HasherMock{} + dataComponents.DataPool = pool + dataComponents.Storage = initStore() + bootstrapComponents.Coordinator = mock.NewMultiShardsCoordinatorMock(noOfShards) + arguments := createMockMetaArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + + startHeaders := createGenesisBlocks(bootstrapComponents.ShardCoordinator()) + arguments.BlockTracker = mock.NewBlockTrackerMock(bootstrapComponents.ShardCoordinator(), startHeaders) + + received := make(map[uint32]uint64) + var mu sync.Mutex + arguments.MiniBlockTracker = &testscommon.MiniBlockTrackerStub{ + ReleaseImmunityForCommittedShardBlocksCalled: func(senderShard uint32, threshold uint64) { + mu.Lock() + received[senderShard] = threshold + mu.Unlock() + }, + } + + mp, err := blproc.NewMetaProcessor(arguments) + require.Nil(t, err) + + const baseNonce = uint64(44) + setLastNotarizedHdr(noOfShards, 9, baseNonce, []byte("randseed"), mp.NotarizedHdrs(), arguments.BlockTracker) + + err = mp.SaveLastNotarizedHeader(&block.MetaBlock{}) + require.Nil(t, err) + + mu.Lock() + defer mu.Unlock() + require.Equal(t, int(noOfShards), len(received), "every shard should receive a release") + for shardID := uint32(0); shardID < noOfShards; shardID++ { + require.Equal(t, baseNonce+1, received[shardID], "shard %d expected hdr.GetNonce()+1", shardID) + } +} + func TestMetaProcessor_CheckShardHeadersValidity(t *testing.T) { t.Parallel() diff --git a/process/block/shardblock_test.go b/process/block/shardblock_test.go index 24051d6f7b1..0bacb9c6a59 100644 --- a/process/block/shardblock_test.go +++ b/process/block/shardblock_test.go @@ -3631,6 +3631,86 @@ func TestShardProcessor_RemoveAndSaveLastNotarizedMetaHdrNoDstMB(t *testing.T) { assert.Equal(t, currHdr, sp.LastNotarizedHdrForShard(core.MetachainShardId)) } +func TestShardProcessor_SaveLastNotarizedHeader_ReleasesImmunityForCommittedMetaBlocks(t *testing.T) { + t.Parallel() + + t.Run("processed headers advance the last cross-notarized nonce", func(t *testing.T) { + var received uint64 + var called atomicCore.Flag + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + startHeaders := createGenesisBlocks(bootstrapComponents.ShardCoordinator()) + arguments.BlockTracker = mock.NewBlockTrackerMock(bootstrapComponents.ShardCoordinator(), startHeaders) + arguments.MiniBlockTracker = &testscommon.MiniBlockTrackerStub{ + ReleaseImmunityForCommittedMetaBlocksCalled: func(threshold uint64) { + atomic.StoreUint64(&received, threshold) + called.SetValue(true) + }, + } + + sp, err := blproc.NewShardProcessor(arguments) + require.Nil(t, err) + + arguments.BlockTracker.AddCrossNotarizedHeader(core.MetachainShardId, &block.MetaBlock{Nonce: 10}, nil) + + processedHdrs := []data.HeaderHandler{ + &block.MetaBlock{Nonce: 12}, + &block.MetaBlock{Nonce: 15}, + } + err = sp.SaveLastNotarizedHeader(core.MetachainShardId, processedHdrs) + require.Nil(t, err) + require.True(t, called.IsSet()) + require.Equal(t, uint64(16), atomic.LoadUint64(&received)) + }) + + t.Run("no processed headers still releases against the existing cross-notarized nonce", func(t *testing.T) { + var received uint64 + var called atomicCore.Flag + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + startHeaders := createGenesisBlocks(bootstrapComponents.ShardCoordinator()) + arguments.BlockTracker = mock.NewBlockTrackerMock(bootstrapComponents.ShardCoordinator(), startHeaders) + arguments.MiniBlockTracker = &testscommon.MiniBlockTrackerStub{ + ReleaseImmunityForCommittedMetaBlocksCalled: func(threshold uint64) { + atomic.StoreUint64(&received, threshold) + called.SetValue(true) + }, + } + + sp, err := blproc.NewShardProcessor(arguments) + require.Nil(t, err) + + arguments.BlockTracker.AddCrossNotarizedHeader(core.MetachainShardId, &block.MetaBlock{Nonce: 7}, nil) + + err = sp.SaveLastNotarizedHeader(core.MetachainShardId, nil) + require.Nil(t, err) + require.True(t, called.IsSet()) + require.Equal(t, uint64(8), atomic.LoadUint64(&received)) + }) + + t.Run("non-meta shard does not invoke the hook", func(t *testing.T) { + var called atomicCore.Flag + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + startHeaders := createGenesisBlocks(bootstrapComponents.ShardCoordinator()) + arguments.BlockTracker = mock.NewBlockTrackerMock(bootstrapComponents.ShardCoordinator(), startHeaders) + arguments.MiniBlockTracker = &testscommon.MiniBlockTrackerStub{ + ReleaseImmunityForCommittedMetaBlocksCalled: func(_ uint64) { + called.SetValue(true) + }, + } + + sp, err := blproc.NewShardProcessor(arguments) + require.Nil(t, err) + + arguments.BlockTracker.AddCrossNotarizedHeader(0, &block.Header{Nonce: 4}, nil) + + err = sp.SaveLastNotarizedHeader(0, []data.HeaderHandler{&block.Header{Nonce: 5}}) + require.Nil(t, err) + require.False(t, called.IsSet()) + }) +} + func createShardData(hasher hashing.Hasher, marshalizer marshal.Marshalizer, miniBlocks []block.MiniBlock) []block.ShardData { shardData := make([]block.ShardData, len(miniBlocks)) for i := 0; i < len(miniBlocks); i++ { diff --git a/testscommon/cache/immunityCacheSpy.go b/testscommon/cache/immunityCacheSpy.go new file mode 100644 index 00000000000..1f31bb273ce --- /dev/null +++ b/testscommon/cache/immunityCacheSpy.go @@ -0,0 +1,21 @@ +package cache + +// ImmunityCacheSpy is a spy for the ImmunityCache +type ImmunityCacheSpy struct { + *CacherStub + SetOldestImmuneNonceCalled func(uint64) +} + +// ImmunizeKeys is a spy for the ImmunizeKeys method of the ImmunityCache +func (c *ImmunityCacheSpy) ImmunizeKeys(_ [][]byte, _ uint64) (int, int) { return 0, 0 } + +// SetOldestImmuneNonce is a spy for the SetOldestImmuneNonce method of the ImmunityCache +func (c *ImmunityCacheSpy) SetOldestImmuneNonce(nonce uint64) { + if c.SetOldestImmuneNonceCalled != nil { + c.SetOldestImmuneNonceCalled(nonce) + } +} + +func (c *ImmunityCacheSpy) RemoveWithResult(_ []byte) bool { return false } +func (c *ImmunityCacheSpy) NumBytes() int { return 0 } +func (c *ImmunityCacheSpy) Diagnose(_ bool) {} From bbf878f971540ad11944b8d40911ae49c0cd53e1 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Mon, 18 May 2026 15:25:32 +0300 Subject: [PATCH 058/116] refactor mb checks func --- process/block/baseProcess.go | 11 ----------- process/block/metablock.go | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 55a52b59313..d02c91c4965 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -2426,14 +2426,3 @@ func (bp *baseProcessor) checkReceivedProofIfAttestingIsNeeded(proof data.Header bp.chRcvAllHdrs <- true } } - -func (bp *baseProcessor) verifyNonEpochStartMiniBlocks(header data.HeaderHandler) error { - for _, miniBlockHeader := range header.GetMiniBlockHeaderHandlers() { - if miniBlockHeader.GetTypeInt32() == int32(block.RewardsBlock) || - miniBlockHeader.GetTypeInt32() == int32(block.PeerBlock) { - return process.ErrInvalidMiniBlockType - } - } - - return nil -} diff --git a/process/block/metablock.go b/process/block/metablock.go index 33d09c1cec1..bc5d6cbee84 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -562,6 +562,17 @@ func (mp *metaProcessor) verifyEpochStartMiniBlocks(metaBlock *block.MetaBlock) return nil } +func (mp *metaProcessor) verifyNonEpochStartMiniBlocks(header data.HeaderHandler) error { + for _, miniBlockHeader := range header.GetMiniBlockHeaderHandlers() { + if miniBlockHeader.GetTypeInt32() == int32(block.RewardsBlock) || + miniBlockHeader.GetTypeInt32() == int32(block.PeerBlock) { + return process.ErrInvalidMiniBlockType + } + } + + return nil +} + // SetNumProcessedObj will set the num of processed headers func (mp *metaProcessor) SetNumProcessedObj(numObj uint64) { mp.headersCounter.shardMBHeadersTotalProcessed = numObj From 9c9bc75b46697d93165184443949043d1dbdb258 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 19 May 2026 11:54:04 +0300 Subject: [PATCH 059/116] fixes after review --- process/track/miniBlockTrack.go | 38 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/process/track/miniBlockTrack.go b/process/track/miniBlockTrack.go index 8b8225d0283..c7cd47dad27 100644 --- a/process/track/miniBlockTrack.go +++ b/process/track/miniBlockTrack.go @@ -102,12 +102,7 @@ func (mbt *miniBlockTrack) receivedMiniBlock(key []byte, value interface{}) { return } - confirmationInfo, ok := mbt.getConfirmedMiniBlockInfo(key) - if !ok { - return - } - - mbt.immunizeMiniBlock(key, miniBlock, confirmationInfo) + mbt.immunizeMiniBlock(key, miniBlock) } func (mbt *miniBlockTrack) getTransactionPool(mbType block.Type) dataRetriever.ShardedDataCacherNotifier { @@ -190,11 +185,11 @@ func (mbt *miniBlockTrack) registerFromMiniBlockHeaders( // Threshold advance is deferred to commit (see ReleaseImmunityForCommittedMetaBlocks). // Advancing here would release items from older metablocks before this shard executes them. mbt.storeConfirmedMiniBlockInfo(miniBlockHeader.GetHash(), mbInfo) - mbt.tryProcessStoredMiniBlock(miniBlockHeader.GetHash(), mbInfo) + mbt.tryProcessStoredMiniBlock(miniBlockHeader.GetHash()) } } -func (mbt *miniBlockTrack) tryProcessStoredMiniBlock(miniBlockHash []byte, confirmationInfo confirmedMiniBlockInfo) { +func (mbt *miniBlockTrack) tryProcessStoredMiniBlock(miniBlockHash []byte) { value, ok := mbt.miniBlocksPool.Peek(miniBlockHash) if !ok { return @@ -205,19 +200,24 @@ func (mbt *miniBlockTrack) tryProcessStoredMiniBlock(miniBlockHash []byte, confi return } - mbt.immunizeMiniBlock(miniBlockHash, miniBlock, confirmationInfo) + mbt.immunizeMiniBlock(miniBlockHash, miniBlock) } -func (mbt *miniBlockTrack) immunizeMiniBlock(miniBlockHash []byte, miniBlock *block.MiniBlock, confirmationInfo confirmedMiniBlockInfo) { +func (mbt *miniBlockTrack) immunizeMiniBlock(miniBlockHash []byte, miniBlock *block.MiniBlock) { // TODO - stop reusing miniBlock.TxHashes for peer changes, add new fields transactionPool := mbt.getTransactionPool(miniBlock.Type) if check.IfNil(transactionPool) { return } + confirmationInfo, ok := mbt.getConfirmedMiniBlockInfo(miniBlockHash) + if !ok { + return + } + mbt.whitelistHandler.Add(miniBlock.TxHashes) transactionPool.ImmunizeSetOfDataAgainstEviction(miniBlock.TxHashes, confirmationInfo.cacheID, confirmationInfo.nonce) - mbt.removeConfirmedMiniBlockInfo(miniBlockHash) + mbt.removeConfirmedMiniBlockInfo(miniBlockHash, confirmationInfo.nonce) } func (mbt *miniBlockTrack) storeConfirmedMiniBlockInfo(miniBlockHash []byte, info confirmedMiniBlockInfo) { @@ -241,10 +241,20 @@ func (mbt *miniBlockTrack) getConfirmedMiniBlockInfo(miniBlockHash []byte) (conf return info, ok } -func (mbt *miniBlockTrack) removeConfirmedMiniBlockInfo(miniBlockHash []byte) { +func (mbt *miniBlockTrack) removeConfirmedMiniBlockInfo(miniBlockHash []byte, nonce uint64) { mbt.mutConfirmedMiniBlocks.Lock() - delete(mbt.confirmedMiniBlocks, string(miniBlockHash)) - mbt.mutConfirmedMiniBlocks.Unlock() + defer mbt.mutConfirmedMiniBlocks.Unlock() + + key := string(miniBlockHash) + info, ok := mbt.confirmedMiniBlocks[key] + if !ok { + return + } + if info.nonce > nonce { + return + } + + delete(mbt.confirmedMiniBlocks, key) } // CleanupConfirmedMiniBlocksBelow drops every tracked confirmation whose nonce From e9fb0441b237835f635413cad65b5ea2ac9a6719 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 19 May 2026 13:03:14 +0300 Subject: [PATCH 060/116] mb ordering cross check --- process/block/baseProcess.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index d02c91c4965..aacad5d49e3 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -970,7 +970,7 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini mbHashStr := string(mbHash) mbHdr, ok := mbHashesFromHdr[mbHashStr] - if !ok { + if !ok || !bytes.Equal(miniBlockHeaders[i].GetHash(), mbHash) { return process.ErrHeaderBodyMismatch } From 555a84266ccd21bbd14353f81633be40ba50edb2 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Tue, 19 May 2026 13:03:20 +0300 Subject: [PATCH 061/116] extra check for first tx processed --- process/block/baseProcess.go | 28 ++++++++ process/block/baseProcess_test.go | 114 ++++++++++++++++++++++++++++++ process/errors.go | 3 + 3 files changed, 145 insertions(+) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 5b6e315f8f9..00ccd324df7 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1000,6 +1000,11 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini return err } + err = bp.checkIndexOfFirstTxProcessedAgainstTracker(mbHdr, mbHash) + if err != nil { + return err + } + delete(mbHashesFromHdr, mbHashStr) } @@ -1018,6 +1023,29 @@ func checkConstructionStateAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler return nil } +func (bp *baseProcessor) checkIndexOfFirstTxProcessedAgainstTracker(mbHdr data.MiniBlockHeaderHandler, miniBlockHash []byte) error { + selfShardID := bp.shardCoordinator.SelfId() + isIncomingCross := mbHdr.GetReceiverShardID() == selfShardID && mbHdr.GetSenderShardID() != selfShardID + if !isIncomingCross { + return nil + } + + processedMiniBlockInfo, _ := bp.processedMiniBlocksTracker.GetProcessedMiniBlockInfo(miniBlockHash) + expectedIndexOfFirstTxProcessed := processedMiniBlockInfo.IndexOfLastTxProcessed + 1 + if mbHdr.GetIndexOfFirstTxProcessed() != expectedIndexOfFirstTxProcessed { + log.Debug("checkIndexOfFirstTxProcessedAgainstTracker: mismatch", + "mb hash", miniBlockHash, + "sender shard", mbHdr.GetSenderShardID(), + "receiver shard", mbHdr.GetReceiverShardID(), + "header index of first tx processed", mbHdr.GetIndexOfFirstTxProcessed(), + "expected index of first tx processed", expectedIndexOfFirstTxProcessed, + ) + return process.ErrIndexOfFirstTxProcessedMismatch + } + + return nil +} + func (bp *baseProcessor) checkScheduledMiniBlocksValidity(headerHandler data.HeaderHandler) error { if !bp.enableEpochsHandler.IsFlagEnabled(common.ScheduledMiniBlocksFlag) { return nil diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index bdbc373e89d..902a465dbb6 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -943,6 +943,120 @@ func TestBaseProcessor_SetIndexOfFirstTxProcessed(t *testing.T) { assert.Equal(t, int32(9), miniBlockHeader.GetIndexOfFirstTxProcessed()) } +func TestBaseProcessor_CheckHeaderBodyCorrelationIndexOfFirstTxProcessed(t *testing.T) { + t.Parallel() + + hasher := &mock.HasherStub{} + marshaller := &mock.MarshalizerMock{} + + t.Run("fresh incoming mb with non-zero IndexOfFirstTxProcessed should error", func(t *testing.T) { + t.Parallel() + + hdr, body := createOneHeaderOneBody() + hdr.MiniBlockHeaders[0].TxCount = 3 + body.MiniBlocks[0].TxHashes = [][]byte{[]byte("tx1"), []byte("tx2"), []byte("tx3")} + mbBytes, _ := marshaller.Marshal(body.MiniBlocks[0]) + hdr.MiniBlockHeaders[0].Hash = hasher.Compute(string(mbBytes)) + _ = hdr.MiniBlockHeaders[0].SetIndexOfFirstTxProcessed(2) + _ = hdr.MiniBlockHeaders[0].SetIndexOfLastTxProcessed(2) + + arguments := CreateMockArguments(createComponentHolderMocks()) + arguments.ProcessedMiniBlocksTracker = processedMb.NewProcessedMiniBlocksTracker() + sp, _ := blproc.NewShardProcessor(arguments) + + err := sp.CheckHeaderBodyCorrelation(hdr, body) + assert.Equal(t, process.ErrIndexOfFirstTxProcessedMismatch, err) + }) + + t.Run("fresh incoming mb with IndexOfFirstTxProcessed=0 should pass", func(t *testing.T) { + t.Parallel() + + hdr, body := createOneHeaderOneBody() + + arguments := CreateMockArguments(createComponentHolderMocks()) + arguments.ProcessedMiniBlocksTracker = processedMb.NewProcessedMiniBlocksTracker() + sp, _ := blproc.NewShardProcessor(arguments) + + err := sp.CheckHeaderBodyCorrelation(hdr, body) + assert.Nil(t, err) + }) + + t.Run("partially processed mb with matching continuation should pass", func(t *testing.T) { + t.Parallel() + + hdr, body := createOneHeaderOneBody() + hdr.MiniBlockHeaders[0].TxCount = 5 + body.MiniBlocks[0].TxHashes = [][]byte{[]byte("tx1"), []byte("tx2"), []byte("tx3"), []byte("tx4"), []byte("tx5")} + mbBytes, _ := marshaller.Marshal(body.MiniBlocks[0]) + mbHash := hasher.Compute(string(mbBytes)) + hdr.MiniBlockHeaders[0].Hash = mbHash + // tracker says we already processed indices 0, 1, 2 so next first must be 3 + _ = hdr.MiniBlockHeaders[0].SetIndexOfFirstTxProcessed(3) + _ = hdr.MiniBlockHeaders[0].SetIndexOfLastTxProcessed(4) + + arguments := CreateMockArguments(createComponentHolderMocks()) + tracker := processedMb.NewProcessedMiniBlocksTracker() + tracker.SetProcessedMiniBlockInfo([]byte("meta_hash"), mbHash, &processedMb.ProcessedMiniBlockInfo{ + FullyProcessed: false, + IndexOfLastTxProcessed: 2, + }) + arguments.ProcessedMiniBlocksTracker = tracker + sp, _ := blproc.NewShardProcessor(arguments) + + err := sp.CheckHeaderBodyCorrelation(hdr, body) + assert.Nil(t, err) + }) + + t.Run("partially processed mb with mismatched continuation should error", func(t *testing.T) { + t.Parallel() + + hdr, body := createOneHeaderOneBody() + hdr.MiniBlockHeaders[0].TxCount = 5 + body.MiniBlocks[0].TxHashes = [][]byte{[]byte("tx1"), []byte("tx2"), []byte("tx3"), []byte("tx4"), []byte("tx5")} + mbBytes, _ := marshaller.Marshal(body.MiniBlocks[0]) + mbHash := hasher.Compute(string(mbBytes)) + hdr.MiniBlockHeaders[0].Hash = mbHash + // tracker says next first must be 3, but proposer forged 0 + _ = hdr.MiniBlockHeaders[0].SetIndexOfFirstTxProcessed(0) + _ = hdr.MiniBlockHeaders[0].SetIndexOfLastTxProcessed(4) + + arguments := CreateMockArguments(createComponentHolderMocks()) + tracker := processedMb.NewProcessedMiniBlocksTracker() + tracker.SetProcessedMiniBlockInfo([]byte("meta_hash"), mbHash, &processedMb.ProcessedMiniBlockInfo{ + FullyProcessed: false, + IndexOfLastTxProcessed: 2, + }) + arguments.ProcessedMiniBlocksTracker = tracker + sp, _ := blproc.NewShardProcessor(arguments) + + err := sp.CheckHeaderBodyCorrelation(hdr, body) + assert.Equal(t, process.ErrIndexOfFirstTxProcessedMismatch, err) + }) + + t.Run("intra shard mb should skip the tracker check", func(t *testing.T) { + t.Parallel() + + hdr, body := createOneHeaderOneBody() + hdr.MiniBlockHeaders[0].TxCount = 3 + body.MiniBlocks[0].TxHashes = [][]byte{[]byte("tx1"), []byte("tx2"), []byte("tx3")} + body.MiniBlocks[0].SenderShardID = 0 + body.MiniBlocks[0].ReceiverShardID = 0 + hdr.MiniBlockHeaders[0].SenderShardID = 0 + hdr.MiniBlockHeaders[0].ReceiverShardID = 0 + mbBytes, _ := marshaller.Marshal(body.MiniBlocks[0]) + hdr.MiniBlockHeaders[0].Hash = hasher.Compute(string(mbBytes)) + _ = hdr.MiniBlockHeaders[0].SetIndexOfFirstTxProcessed(1) + _ = hdr.MiniBlockHeaders[0].SetIndexOfLastTxProcessed(2) + + arguments := CreateMockArguments(createComponentHolderMocks()) + arguments.ProcessedMiniBlocksTracker = processedMb.NewProcessedMiniBlocksTracker() + sp, _ := blproc.NewShardProcessor(arguments) + + err := sp.CheckHeaderBodyCorrelation(hdr, body) + assert.Nil(t, err) + }) +} + func TestBaseProcessor_SetIndexOfLastTxProcessed(t *testing.T) { t.Parallel() diff --git a/process/errors.go b/process/errors.go index dabdef5f176..272c5f718db 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1146,6 +1146,9 @@ var ErrIndexDoesNotMatchWithPartialExecutedMiniBlock = errors.New("index does no // ErrIndexDoesNotMatchWithFullyExecutedMiniBlock signals that the given index does not match with a fully executed mini block var ErrIndexDoesNotMatchWithFullyExecutedMiniBlock = errors.New("index does not match with a fully executed mini block") +// ErrIndexOfFirstTxProcessedMismatch signals that the index of first tx processed from the header does not match the local processed mini blocks tracker +var ErrIndexOfFirstTxProcessedMismatch = errors.New("index of first tx processed does not match the local processed mini blocks tracker") + // ErrNilProcessedMiniBlocksTracker signals that a nil processed mini blocks tracker has been provided var ErrNilProcessedMiniBlocksTracker = errors.New("nil processed mini blocks tracker") From cde8193070d6d70cd84b82ab0b60be8dacb0aef1 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 19 May 2026 14:08:23 +0300 Subject: [PATCH 062/116] fixes failing tests --- consensus/spos/consensusMessageValidator.go | 7 +- .../resolvers/miniblocks/miniblocks_test.go | 66 ++++++++++--------- integrationTests/resolvers/testInitializer.go | 7 +- .../consensusNotAchieved_test.go | 7 +- .../headerCheck/headerSignatureVerify_test.go | 9 +-- 5 files changed, 51 insertions(+), 45 deletions(-) diff --git a/consensus/spos/consensusMessageValidator.go b/consensus/spos/consensusMessageValidator.go index 2c344b10659..1a070d33044 100644 --- a/consensus/spos/consensusMessageValidator.go +++ b/consensus/spos/consensusMessageValidator.go @@ -9,12 +9,13 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/marshal" crypto "github.com/multiversx/mx-chain-crypto-go" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/consensus" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" - logger "github.com/multiversx/mx-chain-logger-go" ) type consensusMessageValidator struct { @@ -493,8 +494,8 @@ func (cmv *consensusMessageValidator) checkMessageWithInvalidSingersValidity(cns } func (cmv *consensusMessageValidator) isMessageTypeLimitReached(pk []byte, round int64, msgType consensus.MessageType) bool { - cmv.mutPkConsensusMessages.RLock() - defer cmv.mutPkConsensusMessages.RUnlock() + cmv.mutPkConsensusMessages.Lock() + defer cmv.mutPkConsensusMessages.Unlock() key := fmt.Sprintf("%s_%d", string(pk), round) diff --git a/integrationTests/resolvers/miniblocks/miniblocks_test.go b/integrationTests/resolvers/miniblocks/miniblocks_test.go index 989dd239ec6..3b94806dc96 100644 --- a/integrationTests/resolvers/miniblocks/miniblocks_test.go +++ b/integrationTests/resolvers/miniblocks/miniblocks_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/integrationTests/resolvers" "github.com/multiversx/mx-chain-go/process/factory" ) @@ -21,12 +23,12 @@ func TestRequestResolveMiniblockByHashRequestingShardResolvingSameShard(t *testi nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardId, shardId) + miniblock, hash := resolvers.CreateMiniblock(shardId, shardId, block.TxBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -34,7 +36,7 @@ func TestRequestResolveMiniblockByHashRequestingShardResolvingSameShard(t *testi } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.IntraShardRequester(factory.MiniBlocksTopic) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -57,12 +59,12 @@ func TestRequestResolveMiniblockByHashRequestingShardResolvingOtherShard(t *test nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardIdResolver, shardIdRequester) + miniblock, hash := resolvers.CreateMiniblock(shardIdResolver, shardIdRequester, block.TxBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -70,7 +72,7 @@ func TestRequestResolveMiniblockByHashRequestingShardResolvingOtherShard(t *test } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, shardIdResolver) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -92,12 +94,12 @@ func TestRequestResolveMiniblockByHashRequestingShardResolvingMeta(t *testing.T) nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardId, shardId) + miniblock, hash := resolvers.CreateMiniblock(shardId, shardId, block.TxBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -105,7 +107,7 @@ func TestRequestResolveMiniblockByHashRequestingShardResolvingMeta(t *testing.T) } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, core.MetachainShardId) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -127,12 +129,12 @@ func TestRequestResolveMiniblockByHashRequestingMetaResolvingShard(t *testing.T) nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardId, core.MetachainShardId) + miniblock, hash := resolvers.CreateMiniblock(shardId, core.MetachainShardId, block.TxBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -140,7 +142,7 @@ func TestRequestResolveMiniblockByHashRequestingMetaResolvingShard(t *testing.T) } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, shardId) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -162,12 +164,12 @@ func TestRequestResolvePeerMiniblockByHashRequestingShardResolvingSameShard(t *t nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(core.MetachainShardId, core.AllShardId) + miniblock, hash := resolvers.CreateMiniblock(core.MetachainShardId, core.AllShardId, block.PeerBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -175,7 +177,7 @@ func TestRequestResolvePeerMiniblockByHashRequestingShardResolvingSameShard(t *t } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, core.AllShardId) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -198,12 +200,12 @@ func TestRequestResolvePeerMiniblockByHashRequestingShardResolvingOtherShard(t * nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardIdResolver, core.AllShardId) + miniblock, hash := resolvers.CreateMiniblock(core.MetachainShardId, core.AllShardId, block.PeerBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -211,7 +213,7 @@ func TestRequestResolvePeerMiniblockByHashRequestingShardResolvingOtherShard(t * } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, core.AllShardId) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -233,12 +235,12 @@ func TestRequestResolvePeerMiniblockByHashRequestingShardResolvingMeta(t *testin nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardId, core.AllShardId) + miniblock, hash := resolvers.CreateMiniblock(core.MetachainShardId, core.AllShardId, block.PeerBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -246,7 +248,7 @@ func TestRequestResolvePeerMiniblockByHashRequestingShardResolvingMeta(t *testin } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, core.AllShardId) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) @@ -268,12 +270,12 @@ func TestRequestResolvePeerMiniblockByHashRequestingMetaResolvingShard(t *testin nRequester.Close() nResolver.Close() }() - miniblock, hash := resolvers.CreateMiniblock(shardId, core.AllShardId) + miniblock, hash := resolvers.CreateMiniblock(core.MetachainShardId, core.AllShardId, block.PeerBlock) - //add miniblock in pool + // add miniblock in pool _, _ = nResolver.DataPool.MiniBlocks().HasOrAdd(hash, miniblock, miniblock.Size()) - //setup header received event + // setup header received event nRequester.DataPool.MiniBlocks().RegisterHandler(func(key []byte, value interface{}) { if bytes.Equal(key, hash) { resolvers.Log.Info("received miniblock", "hash", key) @@ -281,7 +283,7 @@ func TestRequestResolvePeerMiniblockByHashRequestingMetaResolvingShard(t *testin } }, core.UniqueIdentifier()) - //request by hash should work + // request by hash should work requester, err := nRequester.RequestersFinder.CrossShardRequester(factory.MiniBlocksTopic, core.AllShardId) resolvers.Log.LogIfError(err) nRequester.WhiteListHandler.Add([][]byte{hash}) diff --git a/integrationTests/resolvers/testInitializer.go b/integrationTests/resolvers/testInitializer.go index 2910c7590f7..6db05e61a3f 100644 --- a/integrationTests/resolvers/testInitializer.go +++ b/integrationTests/resolvers/testInitializer.go @@ -10,8 +10,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" - "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-logger-go" + + "github.com/multiversx/mx-chain-go/integrationTests" ) // Log - @@ -115,13 +116,13 @@ func CreateMetaHeader(nonce uint64, chainID []byte) (data.HeaderHandler, []byte) } // CreateMiniblock - -func CreateMiniblock(senderShardId uint32, receiverSharId uint32) (*block.MiniBlock, []byte) { +func CreateMiniblock(senderShardId uint32, receiverSharId uint32, mbType block.Type) (*block.MiniBlock, []byte) { dummyTxHash := make([]byte, integrationTests.TestHasher.Size()) miniblock := &block.MiniBlock{ TxHashes: [][]byte{dummyTxHash}, ReceiverShardID: receiverSharId, SenderShardID: senderShardId, - Type: 0, + Type: mbType, } hash, err := core.CalculateHash(integrationTests.TestMarshalizer, integrationTests.TestHasher, miniblock) diff --git a/integrationTests/singleShard/block/consensusNotAchieved/consensusNotAchieved_test.go b/integrationTests/singleShard/block/consensusNotAchieved/consensusNotAchieved_test.go index 560e8f0ae74..0ea5993550c 100644 --- a/integrationTests/singleShard/block/consensusNotAchieved/consensusNotAchieved_test.go +++ b/integrationTests/singleShard/block/consensusNotAchieved/consensusNotAchieved_test.go @@ -10,11 +10,12 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-crypto-go" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/stretchr/testify/assert" + "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/integrationTests/mock" testBlock "github.com/multiversx/mx-chain-go/integrationTests/singleShard/block" - logger "github.com/multiversx/mx-chain-logger-go" - "github.com/stretchr/testify/assert" ) var log = logger.GetOrCreate("consensusNotAchieved") @@ -108,7 +109,7 @@ func TestConsensus_BlockWithoutTwoThirdsPlusOneSignaturesOrWrongBitmapShouldNotB for _, nodes := range nodesMap { integrationTests.UpdateRound(nodes, round) } - bitMapEnough := []byte{11} // 11 = 0b0000 1011 so 3 signatures + bitMapEnough := []byte{0x3} // 0b0000 0011 so 2 signatures body, hdr, _ = proposeBlock(nodesMap[0][0], round, nonce, bitMapEnough) assert.NotNil(t, body) assert.NotNil(t, hdr) diff --git a/process/headerCheck/headerSignatureVerify_test.go b/process/headerCheck/headerSignatureVerify_test.go index a3ea9f874f2..98e56703506 100644 --- a/process/headerCheck/headerSignatureVerify_test.go +++ b/process/headerCheck/headerSignatureVerify_test.go @@ -656,7 +656,7 @@ func TestHeaderSigVerifier_VerifySignatureNotEnoughSigsShouldErr(t *testing.T) { hdrSigVerifier, _ := NewHeaderSigVerifier(args) header := &dataBlock.Header{ - PubKeysBitmap: []byte("A"), + PubKeysBitmap: []byte{0x03}, RandSeed: []byte("randSeed"), PrevRandSeed: []byte("prevRandSeed"), } @@ -687,7 +687,7 @@ func TestHeaderSigVerifier_VerifySignatureOk(t *testing.T) { hdrSigVerifier, _ := NewHeaderSigVerifier(args) header := &dataBlock.Header{ - PubKeysBitmap: []byte("1"), + PubKeysBitmap: []byte{0x01}, PrevRandSeed: []byte("prevRandSeed"), } @@ -726,7 +726,7 @@ func TestHeaderSigVerifier_VerifySignatureNotEnoughSigsShouldErrWhenFallbackThre hdrSigVerifier, _ := NewHeaderSigVerifier(args) header := &dataBlock.MetaBlock{ - PubKeysBitmap: []byte("C"), + PubKeysBitmap: []byte{0x03}, PrevRandSeed: []byte("prevRandSeed"), } @@ -833,9 +833,10 @@ func TestHeaderSigVerifier_VerifySignatureWithEquivalentProofsActivated(t *testi require.Nil(t, err) require.False(t, wasCalled) + var bitmap byte = 1< Date: Tue, 19 May 2026 14:23:03 +0300 Subject: [PATCH 063/116] extra checks scheduled data --- process/block/baseProcess.go | 24 +++++++- process/block/baseProcess_test.go | 96 +++++++++++++++++++++++++++++++ process/block/export_test.go | 5 ++ process/block/shardblock.go | 2 +- process/errors.go | 3 + 5 files changed, 127 insertions(+), 3 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 5b6e315f8f9..80580fcd4a0 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -230,8 +230,8 @@ func (bp *baseProcessor) checkBlockValidity( return nil } -// checkScheduledRootHash checks if the scheduled root hash from the given header is the same with the current user accounts state root hash -func (bp *baseProcessor) checkScheduledRootHash(headerHandler data.HeaderHandler) error { +// checkScheduledData checks if the scheduled data from the given header matches the locally computed scheduled data +func (bp *baseProcessor) checkScheduledData(headerHandler data.HeaderHandler) error { if !bp.enableEpochsHandler.IsFlagEnabled(common.ScheduledMiniBlocksFlag) { return nil } @@ -252,6 +252,26 @@ func (bp *baseProcessor) checkScheduledRootHash(headerHandler data.HeaderHandler return process.ErrScheduledRootHashDoesNotMatch } + scheduledGasAndFees := bp.scheduledTxsExecutionHandler.GetScheduledGasAndFees() + if additionalData.GetScheduledAccumulatedFees().Cmp(scheduledGasAndFees.AccumulatedFees) != 0 || + additionalData.GetScheduledDeveloperFees().Cmp(scheduledGasAndFees.DeveloperFees) != 0 || + additionalData.GetScheduledGasProvided() != scheduledGasAndFees.GasProvided || + additionalData.GetScheduledGasPenalized() != scheduledGasAndFees.GasPenalized || + additionalData.GetScheduledGasRefunded() != scheduledGasAndFees.GasRefunded { + log.Debug("scheduled gas and fees do not match", + "header accumulated fees", additionalData.GetScheduledAccumulatedFees(), + "computed accumulated fees", scheduledGasAndFees.AccumulatedFees, + "header developer fees", additionalData.GetScheduledDeveloperFees(), + "computed developer fees", scheduledGasAndFees.DeveloperFees, + "header gas provided", additionalData.GetScheduledGasProvided(), + "computed gas provided", scheduledGasAndFees.GasProvided, + "header gas penalized", additionalData.GetScheduledGasPenalized(), + "computed gas penalized", scheduledGasAndFees.GasPenalized, + "header gas refunded", additionalData.GetScheduledGasRefunded(), + "computed gas refunded", scheduledGasAndFees.GasRefunded) + return process.ErrScheduledGasAndFeesDoesNotMatch + } + return nil } diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index bdbc373e89d..a2dbc1a1478 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -2361,6 +2361,102 @@ func TestBaseProcessor_ProcessScheduledBlockShouldWork(t *testing.T) { assert.Equal(t, []string{busyIdentifier, idleIdentifier}, busyIdleCalled) // the order is important } +func TestBaseProcessor_CheckScheduledData(t *testing.T) { + t.Parallel() + + scheduledGasAndFees := scheduled.GasAndFees{ + AccumulatedFees: big.NewInt(11), + DeveloperFees: big.NewInt(12), + GasProvided: 13, + GasPenalized: 14, + GasRefunded: 15, + } + + createProcessorAndHeader := func(t *testing.T) (interface { + CheckScheduledData(data.HeaderHandler) error + }, *block.HeaderV2) { t.Helper(); coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks(); coreComponents.EnableEpochsHandlerField = enableEpochsHandlerMock.NewEnableEpochsHandlerStub(common.ScheduledMiniBlocksFlag); arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents); arguments.ArgBaseProcessor.AccountsDB[state.UserAccountsState] = &stateMock.AccountsStub{ + RootHashCalled: func() ([]byte, error) { + return []byte("scheduled-root"), nil + }, + }; arguments.ArgBaseProcessor.ScheduledTxsExecutionHandler = &testscommon.ScheduledTxsExecutionStub{ + GetScheduledGasAndFeesCalled: func() scheduled.GasAndFees { + return scheduledGasAndFees + }, + }; processor, err := blproc.NewShardProcessor(arguments); require.NoError(t, err); header := &block.HeaderV2{ + Header: &block.Header{}, + ScheduledRootHash: []byte("scheduled-root"), + ScheduledAccumulatedFees: big.NewInt(11), + ScheduledDeveloperFees: big.NewInt(12), + ScheduledGasProvided: 13, + ScheduledGasPenalized: 14, + ScheduledGasRefunded: 15, + }; return processor, header } + + t.Run("should work when scheduled data matches", func(t *testing.T) { + t.Parallel() + + processor, header := createProcessorAndHeader(t) + err := processor.CheckScheduledData(header) + + require.NoError(t, err) + }) + + t.Run("should fail when scheduled accumulated fees mismatch", func(t *testing.T) { + t.Parallel() + + processor, header := createProcessorAndHeader(t) + header.ScheduledAccumulatedFees = big.NewInt(111) + + err := processor.CheckScheduledData(header) + + require.ErrorIs(t, err, process.ErrScheduledGasAndFeesDoesNotMatch) + }) + + t.Run("should fail when scheduled developer fees mismatch", func(t *testing.T) { + t.Parallel() + + processor, header := createProcessorAndHeader(t) + header.ScheduledDeveloperFees = big.NewInt(112) + + err := processor.CheckScheduledData(header) + + require.ErrorIs(t, err, process.ErrScheduledGasAndFeesDoesNotMatch) + }) + + t.Run("should fail when scheduled gas provided mismatch", func(t *testing.T) { + t.Parallel() + + processor, header := createProcessorAndHeader(t) + header.ScheduledGasProvided++ + + err := processor.CheckScheduledData(header) + + require.ErrorIs(t, err, process.ErrScheduledGasAndFeesDoesNotMatch) + }) + + t.Run("should fail when scheduled gas penalized mismatch", func(t *testing.T) { + t.Parallel() + + processor, header := createProcessorAndHeader(t) + header.ScheduledGasPenalized++ + + err := processor.CheckScheduledData(header) + + require.ErrorIs(t, err, process.ErrScheduledGasAndFeesDoesNotMatch) + }) + + t.Run("should fail when scheduled gas refunded mismatch", func(t *testing.T) { + t.Parallel() + + processor, header := createProcessorAndHeader(t) + header.ScheduledGasRefunded++ + + err := processor.CheckScheduledData(header) + + require.ErrorIs(t, err, process.ErrScheduledGasAndFeesDoesNotMatch) + }) +} + // get initial fees on first getGasAndFees call and final fees on second call func createFeeHandlerMockForProcessScheduledBlock(initial, final scheduled.GasAndFees) process.TransactionFeeHandler { runCount := 0 diff --git a/process/block/export_test.go b/process/block/export_test.go index d7818ece09a..60b94dc6fdb 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -547,6 +547,11 @@ func (bp *baseProcessor) UpdateState( bp.updateStateStorage(finalHeader, rootHash, prevRootHash, accounts) } +// CheckScheduledData - +func (bp *baseProcessor) CheckScheduledData(headerHandler data.HeaderHandler) error { + return bp.checkScheduledData(headerHandler) +} + // GasAndFeesDelta - func GasAndFeesDelta(initialGasAndFees, finalGasAndFees scheduled.GasAndFees) scheduled.GasAndFees { return gasAndFeesDelta(initialGasAndFees, finalGasAndFees) diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 8b93dc145f2..1a3e0454fdb 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -194,7 +194,7 @@ func (sp *shardProcessor) ProcessBlock( sp.epochNotifier.CheckEpoch(headerHandler) sp.requestHandler.SetEpoch(headerHandler.GetEpoch()) - err = sp.checkScheduledRootHash(headerHandler) + err = sp.checkScheduledData(headerHandler) if err != nil { return err } diff --git a/process/errors.go b/process/errors.go index dabdef5f176..886a457b7c0 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1086,6 +1086,9 @@ var ErrNilTxMaxTotalCostHandler = errors.New("nil transaction max total cost") // ErrScheduledRootHashDoesNotMatch signals that scheduled root hash does not match var ErrScheduledRootHashDoesNotMatch = errors.New("scheduled root hash does not match") +// ErrScheduledGasAndFeesDoesNotMatch signals that scheduled gas and fees do not match +var ErrScheduledGasAndFeesDoesNotMatch = errors.New("scheduled gas and fees do not match") + // ErrNilAdditionalData signals that additional data is nil var ErrNilAdditionalData = errors.New("nil additional data") From 5369616d61ddc59a8c1b29884cf71feb93315222 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 19 May 2026 18:22:02 +0300 Subject: [PATCH 064/116] extra checks construction state and processing type --- process/block/baseProcess.go | 92 +++++++++++- process/block/baseProcess_test.go | 137 ++++++++++++++++++ process/block/export_test.go | 13 +- process/block/metablock.go | 2 +- process/block/preprocess/transactions.go | 2 +- process/block/preprocess/transactions_test.go | 77 ++++++++++ process/block/shardblock.go | 2 +- process/errors.go | 15 ++ 8 files changed, 333 insertions(+), 7 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 5b6e315f8f9..9a76e50fcb2 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -943,7 +943,7 @@ func isPartiallyExecuted( } // check if header has the same miniblocks as presented in body -func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body) error { +func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, blockShardID uint32) error { mbHashesFromHdr := make(map[string]data.MiniBlockHeaderHandler, len(miniBlockHeaders)) for i := 0; i < len(miniBlockHeaders); i++ { mbHashesFromHdr[string(miniBlockHeaders[i].GetHash())] = miniBlockHeaders[i] @@ -995,7 +995,7 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini return err } - err = checkConstructionStateAndIndexesCorrectness(mbHdr) + err = checkConstructionStateProcessingTypeAndIndexesCorrectness(mbHdr, miniBlock, blockShardID) if err != nil { return err } @@ -1018,6 +1018,94 @@ func checkConstructionStateAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler return nil } +// checkConstructionStateProcessingTypeAndIndexesCorrectness validates the (miniBlock, +// miniBlockHeader) pair belonging to a block of shard blockShardID against the legal +// (hdrPT, sender == blockShardID?, allowed state) rows: +// +// Normal, * -> Final +// Scheduled, yes -> Proposed +// Scheduled, no -> Final +// Processed, yes -> Final +// Processed, no -> impossible +// +// It also checks body PT validity, type-vs-scheduling, body-vs-header PT consistency, +// and IndexOfLastTxProcessed vs ConstructionState. +func checkConstructionStateProcessingTypeAndIndexesCorrectness( + mbh data.MiniBlockHeaderHandler, + miniBlock *block.MiniBlock, + blockShardID uint32, +) error { + bodyPT := miniBlock.GetProcessingType() + hdrPT := mbh.GetProcessingType() + mbType := miniBlock.Type + senderIsBlockShard := mbh.GetSenderShardID() == blockShardID + + // Processed is a header-only re-inclusion tag and must never appear on the body. + if bodyPT != int32(block.Normal) && bodyPT != int32(block.Scheduled) { + return fmt.Errorf("%w: body has invalid processing type %d", + process.ErrInvalidMiniBlockProcessingType, bodyPT) + } + + if mbType != block.TxBlock { + if bodyPT != int32(block.Normal) || hdrPT != int32(block.Normal) { + return fmt.Errorf("%w: miniblock type %s cannot be scheduled (body=%d, header=%d)", + process.ErrInvalidMiniBlockProcessingTypeForType, mbType, bodyPT, hdrPT) + } + } + + state := mbh.GetConstructionState() + switch hdrPT { + case int32(block.Normal): + if state != int32(block.Final) { + return fmt.Errorf("%w: Normal header requires Final, got %d", + process.ErrInvalidConstructionState, state) + } + case int32(block.Scheduled): + if bodyPT != int32(block.Scheduled) { + return fmt.Errorf("%w: header=Scheduled requires body=Scheduled, got body=%d", + process.ErrProcessingTypeBodyHeaderMismatch, bodyPT) + } + if senderIsBlockShard { + if state != int32(block.Proposed) { + return fmt.Errorf("%w: Scheduled header at sender shard requires Proposed, got %d", + process.ErrInvalidConstructionState, state) + } + } else { + if state != int32(block.Final) { + return fmt.Errorf("%w: cross-shard incoming Scheduled requires Final, got %d", + process.ErrInvalidConstructionState, state) + } + } + case int32(block.Processed): + if bodyPT != int32(block.Scheduled) { + return fmt.Errorf("%w: header=Processed requires body=Scheduled, got body=%d", + process.ErrProcessingTypeBodyHeaderMismatch, bodyPT) + } + if !senderIsBlockShard { + return fmt.Errorf("%w: Processed header requires sender == blockShard", + process.ErrInvalidMiniBlockShardRole) + } + if state != int32(block.Final) { + return fmt.Errorf("%w: Processed header requires Final, got %d", + process.ErrInvalidConstructionState, state) + } + default: + return fmt.Errorf("%w: unknown header processing type %d", + process.ErrInvalidMiniBlockProcessingType, hdrPT) + } + + lastIdx := mbh.GetIndexOfLastTxProcessed() + finalIdx := int32(mbh.GetTxCount()) - 1 + if state == int32(block.PartialExecuted) && lastIdx == finalIdx { + return process.ErrIndexDoesNotMatchWithPartialExecutedMiniBlock + } + if state != int32(block.PartialExecuted) && lastIdx != finalIdx { + return process.ErrIndexDoesNotMatchWithFullyExecutedMiniBlock + } + + return nil +} + func (bp *baseProcessor) checkScheduledMiniBlocksValidity(headerHandler data.HeaderHandler) error { if !bp.enableEpochsHandler.IsFlagEnabled(common.ScheduledMiniBlocksFlag) { return nil diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index bdbc373e89d..3fc9cdc77f6 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -3121,6 +3121,143 @@ func TestBaseProcessor_checkConstructionStateAndIndexesCorrectness(t *testing.T) assert.Nil(t, err) } +func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) { + t.Parallel() + + const blockShard = uint32(1) + const otherShard = uint32(2) + + makeMb := func(sender, receiver uint32, mbType block.Type, bodyScheduled bool, txCount int) *block.MiniBlock { + mb := &block.MiniBlock{ + SenderShardID: sender, + ReceiverShardID: receiver, + Type: mbType, + TxHashes: make([][]byte, txCount), + } + for i := range mb.TxHashes { + mb.TxHashes[i] = []byte{byte(i)} + } + if bodyScheduled { + reserved, _ := (&block.MiniBlockReserved{ExecutionType: block.Scheduled}).Marshal() + mb.Reserved = reserved + } + return mb + } + + makeMbh := func(mb *block.MiniBlock, hdrPT block.ProcessingType, state block.MiniBlockState, lastIdx int32) *block.MiniBlockHeader { + mbh := &block.MiniBlockHeader{ + SenderShardID: mb.SenderShardID, + ReceiverShardID: mb.ReceiverShardID, + Type: mb.Type, + TxCount: uint32(len(mb.TxHashes)), + } + _ = mbh.SetProcessingType(int32(hdrPT)) + _ = mbh.SetConstructionState(int32(state)) + _ = mbh.SetIndexOfLastTxProcessed(lastIdx) + return mbh + } + + t.Run("legal cells pass", func(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + sender uint32 + receiver uint32 + body bool + hdrPT block.ProcessingType + state block.MiniBlockState + txCount int + lastIdx int32 + }{ + {"normal intra", blockShard, blockShard, false, block.Normal, block.Final, 3, 2}, + {"normal outgoing", blockShard, otherShard, false, block.Normal, block.Final, 3, 2}, + {"normal incoming", otherShard, blockShard, false, block.Normal, block.Final, 3, 2}, + {"normal incoming with scheduled body", otherShard, blockShard, true, block.Normal, block.Final, 3, 2}, + {"scheduled intra", blockShard, blockShard, true, block.Scheduled, block.Proposed, 3, 2}, + {"scheduled outgoing", blockShard, otherShard, true, block.Scheduled, block.Proposed, 3, 2}, + {"scheduled incoming", otherShard, blockShard, true, block.Scheduled, block.Final, 3, 2}, + {"processed intra", blockShard, blockShard, true, block.Processed, block.Final, 3, 2}, + {"processed outgoing", blockShard, otherShard, true, block.Processed, block.Final, 3, 2}, + {"broadcast peer mb", blockShard, core.AllShardId, false, block.Normal, block.Final, 1, 0}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + mb := makeMb(tc.sender, tc.receiver, block.TxBlock, tc.body, tc.txCount) + mbh := makeMbh(mb, tc.hdrPT, tc.state, tc.lastIdx) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.NoError(t, err) + }) + } + }) + + t.Run("scheduled plus partial executed rejected at sender", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, blockShard, block.TxBlock, true, 3) + mbh := makeMbh(mb, block.Scheduled, block.PartialExecuted, 1) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + }) + + t.Run("scheduled plus partial executed rejected at incoming", func(t *testing.T) { + t.Parallel() + mb := makeMb(otherShard, blockShard, block.TxBlock, true, 3) + mbh := makeMbh(mb, block.Scheduled, block.PartialExecuted, 1) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + }) + + t.Run("scheduled body required when header is scheduled", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, blockShard, block.TxBlock, false, 3) + mbh := makeMbh(mb, block.Scheduled, block.Proposed, 2) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrProcessingTypeBodyHeaderMismatch) + }) + + t.Run("processed must have sender equal block shard", func(t *testing.T) { + t.Parallel() + mb := makeMb(otherShard, blockShard, block.TxBlock, true, 3) + mbh := makeMbh(mb, block.Processed, block.Final, 2) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidMiniBlockShardRole) + }) + + t.Run("processed requires scheduled body", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, blockShard, block.TxBlock, false, 3) + mbh := makeMbh(mb, block.Processed, block.Final, 2) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrProcessingTypeBodyHeaderMismatch) + }) + + t.Run("normal incoming with partial state rejected by destination invariant", func(t *testing.T) { + t.Parallel() + mb := makeMb(otherShard, blockShard, block.TxBlock, false, 3) + mbh := makeMbh(mb, block.Normal, block.PartialExecuted, 1) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + }) + + t.Run("non TxBlock cannot be scheduled", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, blockShard, block.SmartContractResultBlock, true, 2) + mbh := makeMbh(mb, block.Scheduled, block.Proposed, 1) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidMiniBlockProcessingTypeForType) + }) + + t.Run("index inconsistency with partial executed", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, blockShard, block.TxBlock, true, 3) + mbh := makeMbh(mb, block.Processed, block.PartialExecuted, 2) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + }) +} + func TestBaseProcessor_ConcurrentCallsNonceOfFirstCommittedBlock(t *testing.T) { t.Parallel() diff --git a/process/block/export_test.go b/process/block/export_test.go index d7818ece09a..1a304b56198 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -333,7 +333,7 @@ func (mp *metaProcessor) CheckShardHeadersFinality(highestNonceHdrs map[uint32]d // CheckHeaderBodyCorrelation - func (mp *metaProcessor) CheckHeaderBodyCorrelation(hdr data.HeaderHandler, body *block.Body) error { - return mp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body) + return mp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body, hdr.GetShardID()) } // IsHdrConstructionValid - @@ -363,7 +363,7 @@ func (sp *shardProcessor) SaveLastNotarizedHeader(shardId uint32, processedHdrs // CheckHeaderBodyCorrelation - func (sp *shardProcessor) CheckHeaderBodyCorrelation(hdr data.HeaderHandler, body *block.Body) error { - return sp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body) + return sp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body, hdr.GetShardID()) } // CheckAndRequestIfMetaHeadersMissing - @@ -673,6 +673,15 @@ func (bp *baseProcessor) CheckConstructionStateAndIndexesCorrectness(mbh data.Mi return checkConstructionStateAndIndexesCorrectness(mbh) } +// CheckConstructionStateProcessingTypeAndIndexesCorrectness - +func CheckConstructionStateProcessingTypeAndIndexesCorrectness( + mbh data.MiniBlockHeaderHandler, + miniBlock *block.MiniBlock, + blockShardID uint32, +) error { + return checkConstructionStateProcessingTypeAndIndexesCorrectness(mbh, miniBlock, blockShardID) +} + // GetAllMarshalledTxs - func (mp *metaProcessor) GetAllMarshalledTxs(body *block.Body) map[string][][]byte { return mp.getAllMarshalledTxs(body) diff --git a/process/block/metablock.go b/process/block/metablock.go index 33f0b4ac917..9ed5359c5a1 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -239,7 +239,7 @@ func (mp *metaProcessor) ProcessBlock( return process.ErrWrongTypeAssertion } - err = mp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body) + err = mp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) if err != nil { return err } diff --git a/process/block/preprocess/transactions.go b/process/block/preprocess/transactions.go index 2a724927bbd..c7d5e8b6a7a 100644 --- a/process/block/preprocess/transactions.go +++ b/process/block/preprocess/transactions.go @@ -1562,7 +1562,7 @@ func (txs *transactions) ProcessMiniBlock( numTXsProcessed++ } - if err != nil && !partialMbExecutionMode { + if err != nil && (!partialMbExecutionMode || scheduledMode) { return processedTxHashes, txIndex - 1, true, err } diff --git a/process/block/preprocess/transactions_test.go b/process/block/preprocess/transactions_test.go index 68dd2d7e709..e3c6a799027 100644 --- a/process/block/preprocess/transactions_test.go +++ b/process/block/preprocess/transactions_test.go @@ -1393,6 +1393,83 @@ func TestTransactionsPreprocessor_ProcessMiniBlockShouldErrMaxGasLimitUsedForDes assert.Equal(t, -1, indexOfLastTxProcessed) } +// Scheduled mode mid-MB break must always roll back the whole MB, even when +// partial-execution mode is on, so the resulting header never combines +// ProcessingType=Scheduled with ConstructionState=PartialExecuted. +func TestTransactionsPreprocessor_ProcessMiniBlockScheduledRollsBackOnError(t *testing.T) { + t.Parallel() + + tdp := &dataRetrieverMock.PoolsHolderStub{ + TransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return &testscommon.ShardedDataStub{ + ShardDataStoreCalled: func(id string) (c storage.Cacher) { + return &cache.CacherStub{ + PeekCalled: func(key []byte) (value interface{}, ok bool) { + return &transaction.Transaction{}, true + }, + } + }, + } + }, + } + + txHashes := [][]byte{[]byte("tx_hash1"), []byte("tx_hash2")} + miniBlock := &block.MiniBlock{ + ReceiverShardID: 0, + SenderShardID: 1, + TxHashes: txHashes, + Type: block.TxBlock, + } + preProcessorExecutionInfoHandlerMock := &testscommon.PreProcessorExecutionInfoHandlerMock{ + GetNumOfCrossInterMbsAndTxsCalled: getNumOfCrossInterMbsAndTxsZero, + } + + // haveTime returns true for the initial getAllTxsFromMiniBlock per-tx checks + // (one call per tx hash), then false so the per-tx processing loop breaks + // immediately with ErrTimeIsOut. haveAdditionalTime always returns false so + // both branches of the loop's time-out guard fail together. + makeHaveTimeAllowingFetch := func() func() bool { + remaining := len(txHashes) + return func() bool { + if remaining > 0 { + remaining-- + return true + } + return false + } + } + + cases := []struct { + name string + scheduledMode bool + partialMode bool + expectShouldRevert bool + }{ + {"non-scheduled non-partial revert", false, false, true}, + {"non-scheduled partial do not revert", false, true, false}, + {"scheduled non-partial revert", true, false, true}, + {"scheduled partial revert (mutual exclusion fix)", true, true, true}, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + args := createDefaultTransactionsProcessorArgs() + args.TxDataPool = tdp.Transactions() + txs, err := NewTransactionPreprocessor(args) + require.NoError(t, err) + + _, _, shouldRevert, processErr := txs.ProcessMiniBlock( + miniBlock, makeHaveTimeAllowingFetch(), haveAdditionalTimeFalse, + tc.scheduledMode, tc.partialMode, -1, + preProcessorExecutionInfoHandlerMock, + ) + require.ErrorIs(t, processErr, process.ErrTimeIsOut) + require.Equal(t, tc.expectShouldRevert, shouldRevert) + }) + } +} + func TestTransactionsPreprocessor_ComputeGasProvidedShouldWork(t *testing.T) { t.Parallel() diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 8b93dc145f2..0367769b06e 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -218,7 +218,7 @@ func (sp *shardProcessor) ProcessBlock( go getMetricsFromBlockBody(body, sp.marshalizer, sp.appStatusHandler) - err = sp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body) + err = sp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) if err != nil { return err } diff --git a/process/errors.go b/process/errors.go index dabdef5f176..2c98b75a591 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1146,6 +1146,21 @@ var ErrIndexDoesNotMatchWithPartialExecutedMiniBlock = errors.New("index does no // ErrIndexDoesNotMatchWithFullyExecutedMiniBlock signals that the given index does not match with a fully executed mini block var ErrIndexDoesNotMatchWithFullyExecutedMiniBlock = errors.New("index does not match with a fully executed mini block") +// ErrInvalidMiniBlockProcessingType signals that a miniblock has an invalid ProcessingType value +var ErrInvalidMiniBlockProcessingType = errors.New("invalid miniblock processing type") + +// ErrInvalidMiniBlockProcessingTypeForType signals that the ProcessingType is not allowed for the miniblock's Type +var ErrInvalidMiniBlockProcessingTypeForType = errors.New("invalid miniblock processing type for miniblock type") + +// ErrProcessingTypeBodyHeaderMismatch signals that the ProcessingType in the body and the header do not agree +var ErrProcessingTypeBodyHeaderMismatch = errors.New("processing type mismatch between miniblock body and miniblock header") + +// ErrInvalidConstructionState signals that the ConstructionState is not allowed given the ProcessingType and shard role +var ErrInvalidConstructionState = errors.New("invalid construction state for the given processing type and shard role") + +// ErrInvalidMiniBlockShardRole signals that the miniblock's shard role is not allowed for the given ProcessingType +var ErrInvalidMiniBlockShardRole = errors.New("invalid miniblock shard role for the given processing type") + // ErrNilProcessedMiniBlocksTracker signals that a nil processed mini blocks tracker has been provided var ErrNilProcessedMiniBlocksTracker = errors.New("nil processed mini blocks tracker") From 8df01a1fe3bf8ecd4144f250b35bee1fe8149332 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 20 May 2026 11:27:20 +0300 Subject: [PATCH 065/116] fixes processing type and construction state checks --- process/block/baseProcess.go | 36 ++++++++++++++++--------------- process/block/baseProcess_test.go | 3 ++- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 9a76e50fcb2..c3f67016be5 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1028,8 +1028,9 @@ func checkConstructionStateAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler // Processed, yes -> Final // Processed, no -> impossible // -// It also checks body PT validity, type-vs-scheduling, body-vs-header PT consistency, -// and IndexOfLastTxProcessed vs ConstructionState. +// It also checks body PT validity, type-vs-scheduling, and IndexOfLastTxProcessed vs +// ConstructionState. Body-vs-header PT consistency is enforced only when sender is +// blockShardID; for incoming MBs the body PT belongs to the source shard. func checkConstructionStateProcessingTypeAndIndexesCorrectness( mbh data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, @@ -1053,27 +1054,28 @@ func checkConstructionStateProcessingTypeAndIndexesCorrectness( } } - state := mbh.GetConstructionState() + constructionState := mbh.GetConstructionState() switch hdrPT { case int32(block.Normal): - if state != int32(block.Final) { + if constructionState != int32(block.Final) { return fmt.Errorf("%w: Normal header requires Final, got %d", - process.ErrInvalidConstructionState, state) + process.ErrInvalidConstructionState, constructionState) } case int32(block.Scheduled): - if bodyPT != int32(block.Scheduled) { - return fmt.Errorf("%w: header=Scheduled requires body=Scheduled, got body=%d", - process.ErrProcessingTypeBodyHeaderMismatch, bodyPT) - } if senderIsBlockShard { - if state != int32(block.Proposed) { + if bodyPT != int32(block.Scheduled) { + return fmt.Errorf("%w: header=Scheduled requires body=Scheduled, got body=%d", + process.ErrProcessingTypeBodyHeaderMismatch, bodyPT) + } + if constructionState != int32(block.Proposed) { return fmt.Errorf("%w: Scheduled header at sender shard requires Proposed, got %d", - process.ErrInvalidConstructionState, state) + process.ErrInvalidConstructionState, constructionState) } } else { - if state != int32(block.Final) { + // incoming body PT belongs to the source shard, so it is not constrained here + if constructionState != int32(block.Final) { return fmt.Errorf("%w: cross-shard incoming Scheduled requires Final, got %d", - process.ErrInvalidConstructionState, state) + process.ErrInvalidConstructionState, constructionState) } } case int32(block.Processed): @@ -1085,9 +1087,9 @@ func checkConstructionStateProcessingTypeAndIndexesCorrectness( return fmt.Errorf("%w: Processed header requires sender == blockShard", process.ErrInvalidMiniBlockShardRole) } - if state != int32(block.Final) { + if constructionState != int32(block.Final) { return fmt.Errorf("%w: Processed header requires Final, got %d", - process.ErrInvalidConstructionState, state) + process.ErrInvalidConstructionState, constructionState) } default: return fmt.Errorf("%w: unknown header processing type %d", @@ -1096,10 +1098,10 @@ func checkConstructionStateProcessingTypeAndIndexesCorrectness( lastIdx := mbh.GetIndexOfLastTxProcessed() finalIdx := int32(mbh.GetTxCount()) - 1 - if state == int32(block.PartialExecuted) && lastIdx == finalIdx { + if constructionState == int32(block.PartialExecuted) && lastIdx == finalIdx { return process.ErrIndexDoesNotMatchWithPartialExecutedMiniBlock } - if state != int32(block.PartialExecuted) && lastIdx != finalIdx { + if constructionState != int32(block.PartialExecuted) && lastIdx != finalIdx { return process.ErrIndexDoesNotMatchWithFullyExecutedMiniBlock } diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index 3fc9cdc77f6..7e118a90206 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -3176,7 +3176,8 @@ func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) {"normal incoming with scheduled body", otherShard, blockShard, true, block.Normal, block.Final, 3, 2}, {"scheduled intra", blockShard, blockShard, true, block.Scheduled, block.Proposed, 3, 2}, {"scheduled outgoing", blockShard, otherShard, true, block.Scheduled, block.Proposed, 3, 2}, - {"scheduled incoming", otherShard, blockShard, true, block.Scheduled, block.Final, 3, 2}, + {"scheduled incoming with scheduled body", otherShard, blockShard, true, block.Scheduled, block.Final, 3, 2}, + {"scheduled incoming with normal body", otherShard, blockShard, false, block.Scheduled, block.Final, 3, 2}, {"processed intra", blockShard, blockShard, true, block.Processed, block.Final, 3, 2}, {"processed outgoing", blockShard, otherShard, true, block.Processed, block.Final, 3, 2}, {"broadcast peer mb", blockShard, core.AllShardId, false, block.Normal, block.Final, 1, 0}, From 983155ea671ae77a4e408fa51bc6a42c08dc936d Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 20 May 2026 14:25:33 +0300 Subject: [PATCH 066/116] avoid duplicated hashes on request --- .../requestHandlers/requestHandler.go | 9 +++++- .../requestHandlers/requestHandler_test.go | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/dataRetriever/requestHandlers/requestHandler.go b/dataRetriever/requestHandlers/requestHandler.go index 7eae3c93eb3..05f8152ecf3 100644 --- a/dataRetriever/requestHandlers/requestHandler.go +++ b/dataRetriever/requestHandlers/requestHandler.go @@ -787,11 +787,18 @@ func (rrh *resolverRequestHandler) IsInterfaceNil() bool { func (rrh *resolverRequestHandler) getUnrequestedHashes(hashes [][]byte, suffix string) [][]byte { unrequestedHashes := make([][]byte, 0) + seen := make(map[string]struct{}, len(hashes)) rrh.sweepIfNeeded() for _, hash := range hashes { - if !rrh.requestedItemsHandler.Has(string(hash) + suffix) { + key := string(hash) + suffix + if _, alreadySeen := seen[key]; alreadySeen { + continue + } + seen[key] = struct{}{} + + if !rrh.requestedItemsHandler.Has(key) { unrequestedHashes = append(unrequestedHashes, hash) } } diff --git a/dataRetriever/requestHandlers/requestHandler_test.go b/dataRetriever/requestHandlers/requestHandler_test.go index 825664f84c4..f4922d5d5b7 100644 --- a/dataRetriever/requestHandlers/requestHandler_test.go +++ b/dataRetriever/requestHandlers/requestHandler_test.go @@ -1978,6 +1978,38 @@ func TestResolverRequestHandler_RequestMiniblocks(t *testing.T) { rrh.RequestMiniBlocks(0, [][]byte{[]byte("mbHash")}) }) + t.Run("should deduplicate hashes within the same batch", func(t *testing.T) { + t.Parallel() + + duplicateHash := []byte("mbHash") + numCalls := uint32(0) + var receivedHashes [][]byte + mbRequester := &dataRetrieverMocks.HashSliceRequesterStub{ + RequestDataFromHashArrayCalled: func(hashes [][]byte, epoch uint32) error { + atomic.AddUint32(&numCalls, 1) + receivedHashes = hashes + return nil + }, + } + rrh, _ := NewResolverRequestHandler( + &dataRetrieverMocks.RequestersFinderStub{ + CrossShardRequesterCalled: func(baseTopic string, crossShard uint32) (dataRetriever.Requester, error) { + return mbRequester, nil + }, + }, + &mock.RequestedItemsHandlerStub{}, + &mock.WhiteListHandlerStub{}, + 100, + 0, + time.Second, + time.Millisecond, + ) + + rrh.RequestMiniBlocks(0, [][]byte{duplicateHash, duplicateHash, duplicateHash}) + assert.Equal(t, uint32(1), atomic.LoadUint32(&numCalls)) + require.Len(t, receivedHashes, 1) + assert.Equal(t, duplicateHash, receivedHashes[0]) + }) } func TestResolverRequestHandler_RequestInterval(t *testing.T) { From 5a15dc222dc0613af072ad971d8d6e202529292c Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 20 May 2026 14:29:12 +0300 Subject: [PATCH 067/116] update go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a837dde8cd8..621fca50db5 100644 --- a/go.mod +++ b/go.mod @@ -209,4 +209,4 @@ require ( replace github.com/gogo/protobuf => github.com/multiversx/protobuf v1.3.2 -replace github.com/multiversx/mx-chain-storage-go => github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260515145423-6fa5d611f6b6 +replace github.com/multiversx/mx-chain-storage-go => github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a diff --git a/go.sum b/go.sum index 3573ff4102d..7bea6b0a8f7 100644 --- a/go.sum +++ b/go.sum @@ -411,8 +411,8 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260515145423-6fa5d611f6b6 h1:cVkUgwm0W+egegdlRbDi4tnZB6hLmgNl9kT7uzRskSI= -github.com/multiversx/mx-chain-storage-go-private v0.0.0-20260515145423-6fa5d611f6b6/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a h1:zCJjTv47zA6+1s9aNTqubfd8ntOtoiBBZDEwLOoN4Z8= +github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= From 957d3cd75d36487b82796f3705a2fd02bde5ef1c Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 20 May 2026 14:30:54 +0300 Subject: [PATCH 068/116] fixes processing type and construction state checks --- process/block/baseProcess.go | 15 ++++++++------- process/block/baseProcess_test.go | 8 ++++---- process/block/export_test.go | 6 +----- process/errors.go | 10 +++++----- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index c3f67016be5..08e481184a1 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1020,11 +1020,12 @@ func checkConstructionStateAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler // checkConstructionStateProcessingTypeAndIndexesCorrectness validates the (miniBlock, // miniBlockHeader) pair belonging to a block of shard blockShardID against the legal -// (hdrPT, sender == blockShardID?, allowed state) rows: +// (hdrPT, sender == blockShardID?, allowed state) rows. PartialExecuted is allowed alongside +// the primary state of each processing type, validated by the index check: // // Normal, * -> Final -// Scheduled, yes -> Proposed -// Scheduled, no -> Final +// Scheduled, yes -> Proposed | PartialExecuted +// Scheduled, no -> Final | PartialExecuted // Processed, yes -> Final // Processed, no -> impossible // @@ -1067,14 +1068,14 @@ func checkConstructionStateProcessingTypeAndIndexesCorrectness( return fmt.Errorf("%w: header=Scheduled requires body=Scheduled, got body=%d", process.ErrProcessingTypeBodyHeaderMismatch, bodyPT) } - if constructionState != int32(block.Proposed) { - return fmt.Errorf("%w: Scheduled header at sender shard requires Proposed, got %d", + if constructionState != int32(block.Proposed) && constructionState != int32(block.PartialExecuted) { + return fmt.Errorf("%w: Scheduled header at sender shard requires Proposed or PartialExecuted, got %d", process.ErrInvalidConstructionState, constructionState) } } else { // incoming body PT belongs to the source shard, so it is not constrained here - if constructionState != int32(block.Final) { - return fmt.Errorf("%w: cross-shard incoming Scheduled requires Final, got %d", + if constructionState != int32(block.Final) && constructionState != int32(block.PartialExecuted) { + return fmt.Errorf("%w: cross-shard incoming Scheduled requires Final or PartialExecuted, got %d", process.ErrInvalidConstructionState, constructionState) } } diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index 7e118a90206..8c611289a5a 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -3194,20 +3194,20 @@ func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) } }) - t.Run("scheduled plus partial executed rejected at sender", func(t *testing.T) { + t.Run("scheduled plus partial executed allowed at sender", func(t *testing.T) { t.Parallel() mb := makeMb(blockShard, blockShard, block.TxBlock, true, 3) mbh := makeMbh(mb, block.Scheduled, block.PartialExecuted, 1) err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) - assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + assert.NoError(t, err) }) - t.Run("scheduled plus partial executed rejected at incoming", func(t *testing.T) { + t.Run("scheduled plus partial executed allowed at incoming", func(t *testing.T) { t.Parallel() mb := makeMb(otherShard, blockShard, block.TxBlock, true, 3) mbh := makeMbh(mb, block.Scheduled, block.PartialExecuted, 1) err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) - assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + assert.NoError(t, err) }) t.Run("scheduled body required when header is scheduled", func(t *testing.T) { diff --git a/process/block/export_test.go b/process/block/export_test.go index 1a304b56198..97407096d53 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -674,11 +674,7 @@ func (bp *baseProcessor) CheckConstructionStateAndIndexesCorrectness(mbh data.Mi } // CheckConstructionStateProcessingTypeAndIndexesCorrectness - -func CheckConstructionStateProcessingTypeAndIndexesCorrectness( - mbh data.MiniBlockHeaderHandler, - miniBlock *block.MiniBlock, - blockShardID uint32, -) error { +func CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { return checkConstructionStateProcessingTypeAndIndexesCorrectness(mbh, miniBlock, blockShardID) } diff --git a/process/errors.go b/process/errors.go index 2c98b75a591..f712d3e4728 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1146,19 +1146,19 @@ var ErrIndexDoesNotMatchWithPartialExecutedMiniBlock = errors.New("index does no // ErrIndexDoesNotMatchWithFullyExecutedMiniBlock signals that the given index does not match with a fully executed mini block var ErrIndexDoesNotMatchWithFullyExecutedMiniBlock = errors.New("index does not match with a fully executed mini block") -// ErrInvalidMiniBlockProcessingType signals that a miniblock has an invalid ProcessingType value +// ErrInvalidMiniBlockProcessingType signals that an invalid miniblock processing type has been provided var ErrInvalidMiniBlockProcessingType = errors.New("invalid miniblock processing type") -// ErrInvalidMiniBlockProcessingTypeForType signals that the ProcessingType is not allowed for the miniblock's Type +// ErrInvalidMiniBlockProcessingTypeForType signals an invalid miniblock processing type for the given miniblock type var ErrInvalidMiniBlockProcessingTypeForType = errors.New("invalid miniblock processing type for miniblock type") -// ErrProcessingTypeBodyHeaderMismatch signals that the ProcessingType in the body and the header do not agree +// ErrProcessingTypeBodyHeaderMismatch signals a processing type mismatch between the miniblock body and its header var ErrProcessingTypeBodyHeaderMismatch = errors.New("processing type mismatch between miniblock body and miniblock header") -// ErrInvalidConstructionState signals that the ConstructionState is not allowed given the ProcessingType and shard role +// ErrInvalidConstructionState signals an invalid construction state for the given processing type and shard role var ErrInvalidConstructionState = errors.New("invalid construction state for the given processing type and shard role") -// ErrInvalidMiniBlockShardRole signals that the miniblock's shard role is not allowed for the given ProcessingType +// ErrInvalidMiniBlockShardRole signals an invalid miniblock shard role for the given processing type var ErrInvalidMiniBlockShardRole = errors.New("invalid miniblock shard role for the given processing type") // ErrNilProcessedMiniBlocksTracker signals that a nil processed mini blocks tracker has been provided From 283fee04f8273ab162ab393c4ab3bb93dce663b5 Mon Sep 17 00:00:00 2001 From: miiu Date: Wed, 20 May 2026 15:38:31 +0300 Subject: [PATCH 069/116] go mod tidy --- go.sum | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 7bea6b0a8f7..2341bedfa9d 100644 --- a/go.sum +++ b/go.sum @@ -401,20 +401,20 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.1 h1:ljs53jpdjtCohpaqm2n/dvTGrFlSgIpoZYH8RVt5cWo= -github.com/multiversx/mx-chain-core-go v1.4.1/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 h1:hpnYOT5cDJip7B6GvFRSOcUdwdhBbuFsNjLvOxzYOn8= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= -github.com/multiversx/mx-chain-es-indexer-go v1.9.2 h1:/K/cpTkwlFJ7zOD8VRhgc6ixi1t/3ua8CLl63LWHjvE= -github.com/multiversx/mx-chain-es-indexer-go v1.9.2/go.mod h1:t1rkD2vHXSI4EClig0h7+kRCSUCRrMF+emr4DHxFtfA= +github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= +github.com/multiversx/mx-chain-es-indexer-go v1.9.3/go.mod h1:dXRu2fmdiLFOcaRA34axQfoUcq8p9NUGqr4+9dN+p0Y= github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/WI0Qh112whgZNM= github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a h1:zCJjTv47zA6+1s9aNTqubfd8ntOtoiBBZDEwLOoN4Z8= github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= -github.com/multiversx/mx-chain-vm-common-go v1.6.5 h1:Uze7oTTsrkbx3QWbAZ00YTpBXX4qyp+mHuxrH2pSCgc= -github.com/multiversx/mx-chain-vm-common-go v1.6.5/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= +github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a h1:arc/Q+8Q8F1GnCGCtJu+sWnxzBmPgqtiIdzNk6tJ1T8= +github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= github.com/multiversx/mx-chain-vm-go v1.5.45/go.mod h1:Qc2Sckw+EfQwnapkzghFfhuUAOGv29oSZgvj8LJ+xWQ= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 h1:5gSR3IMw1mcp/v5oO+vZ5YOyWO8w7O2qKhCKNPwsWNE= From 63d6cb9435df828e09465eecb3b2104c32209aa5 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 20 May 2026 16:07:27 +0300 Subject: [PATCH 070/116] add allowed combination remove unused methods --- process/block/baseProcess.go | 29 ++++++++---------- process/block/baseProcess_test.go | 49 ++++++------------------------- process/block/export_test.go | 5 ---- 3 files changed, 22 insertions(+), 61 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 08e481184a1..a4ae5f124ee 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1006,24 +1006,13 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini return nil } -func checkConstructionStateAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler) error { - if mbh.GetConstructionState() == int32(block.PartialExecuted) && mbh.GetIndexOfLastTxProcessed() == int32(mbh.GetTxCount())-1 { - return process.ErrIndexDoesNotMatchWithPartialExecutedMiniBlock - - } - if mbh.GetConstructionState() != int32(block.PartialExecuted) && mbh.GetIndexOfLastTxProcessed() != int32(mbh.GetTxCount())-1 { - return process.ErrIndexDoesNotMatchWithFullyExecutedMiniBlock - } - - return nil -} - // checkConstructionStateProcessingTypeAndIndexesCorrectness validates the (miniBlock, // miniBlockHeader) pair belonging to a block of shard blockShardID against the legal // (hdrPT, sender == blockShardID?, allowed state) rows. PartialExecuted is allowed alongside // the primary state of each processing type, validated by the index check: // -// Normal, * -> Final +// Normal, yes -> Final +// Normal, no -> Final | PartialExecuted // Scheduled, yes -> Proposed | PartialExecuted // Scheduled, no -> Final | PartialExecuted // Processed, yes -> Final @@ -1058,9 +1047,17 @@ func checkConstructionStateProcessingTypeAndIndexesCorrectness( constructionState := mbh.GetConstructionState() switch hdrPT { case int32(block.Normal): - if constructionState != int32(block.Final) { - return fmt.Errorf("%w: Normal header requires Final, got %d", - process.ErrInvalidConstructionState, constructionState) + if senderIsBlockShard { + if constructionState != int32(block.Final) { + return fmt.Errorf("%w: Normal header at sender shard requires Final, got %d", + process.ErrInvalidConstructionState, constructionState) + } + } else { + // an incoming normal miniblock may be partially executed at the destination + if constructionState != int32(block.Final) && constructionState != int32(block.PartialExecuted) { + return fmt.Errorf("%w: incoming Normal header requires Final or PartialExecuted, got %d", + process.ErrInvalidConstructionState, constructionState) + } } case int32(block.Scheduled): if senderIsBlockShard { diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index 8c611289a5a..52b40055af9 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -3082,45 +3082,6 @@ func TestBaseProcessor_getPruningHandlerSetsDefaulPruningDelay(t *testing.T) { assert.False(t, ph.IsPruningEnabled()) } -func TestBaseProcessor_checkConstructionStateAndIndexesCorrectness(t *testing.T) { - t.Parallel() - - arguments := CreateMockArguments(createComponentHolderMocks()) - bp, _ := blproc.NewShardProcessor(arguments) - - mbh := &block.MiniBlockHeader{ - TxCount: 5, - } - - _ = mbh.SetConstructionState(int32(block.PartialExecuted)) - - _ = mbh.SetIndexOfLastTxProcessed(int32(mbh.TxCount)) - err := bp.CheckConstructionStateAndIndexesCorrectness(mbh) - assert.Nil(t, err) - - _ = mbh.SetIndexOfLastTxProcessed(int32(mbh.TxCount) - 2) - err = bp.CheckConstructionStateAndIndexesCorrectness(mbh) - assert.Nil(t, err) - - _ = mbh.SetIndexOfLastTxProcessed(int32(mbh.TxCount) - 1) - err = bp.CheckConstructionStateAndIndexesCorrectness(mbh) - assert.Equal(t, process.ErrIndexDoesNotMatchWithPartialExecutedMiniBlock, err) - - _ = mbh.SetConstructionState(int32(block.Final)) - - _ = mbh.SetIndexOfLastTxProcessed(int32(mbh.TxCount)) - err = bp.CheckConstructionStateAndIndexesCorrectness(mbh) - assert.Equal(t, process.ErrIndexDoesNotMatchWithFullyExecutedMiniBlock, err) - - _ = mbh.SetIndexOfLastTxProcessed(int32(mbh.TxCount) - 2) - err = bp.CheckConstructionStateAndIndexesCorrectness(mbh) - assert.Equal(t, process.ErrIndexDoesNotMatchWithFullyExecutedMiniBlock, err) - - _ = mbh.SetIndexOfLastTxProcessed(int32(mbh.TxCount) - 1) - err = bp.CheckConstructionStateAndIndexesCorrectness(mbh) - assert.Nil(t, err) -} - func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) { t.Parallel() @@ -3234,11 +3195,19 @@ func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) assert.ErrorIs(t, err, process.ErrProcessingTypeBodyHeaderMismatch) }) - t.Run("normal incoming with partial state rejected by destination invariant", func(t *testing.T) { + t.Run("incoming normal partial executed allowed", func(t *testing.T) { t.Parallel() mb := makeMb(otherShard, blockShard, block.TxBlock, false, 3) mbh := makeMbh(mb, block.Normal, block.PartialExecuted, 1) err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.NoError(t, err) + }) + + t.Run("sender shard normal partial executed rejected", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, otherShard, block.TxBlock, false, 3) + mbh := makeMbh(mb, block.Normal, block.PartialExecuted, 1) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) assert.ErrorIs(t, err, process.ErrInvalidConstructionState) }) diff --git a/process/block/export_test.go b/process/block/export_test.go index 97407096d53..b4508b2d379 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -668,11 +668,6 @@ func (sp *shardProcessor) RollBackProcessedMiniBlocksInfo(headerHandler data.Hea sp.rollBackProcessedMiniBlocksInfo(headerHandler, mapMiniBlockHashes) } -// CheckConstructionStateAndIndexesCorrectness - -func (bp *baseProcessor) CheckConstructionStateAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler) error { - return checkConstructionStateAndIndexesCorrectness(mbh) -} - // CheckConstructionStateProcessingTypeAndIndexesCorrectness - func CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { return checkConstructionStateProcessingTypeAndIndexesCorrectness(mbh, miniBlock, blockShardID) From 245c0e283c71f54fc07de76aaa57ce0bf8ea4fa0 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 20 May 2026 16:52:28 +0300 Subject: [PATCH 071/116] leading scheduled checks --- process/block/baseProcess.go | 34 +++++++++++----- process/block/baseProcess_test.go | 65 +++++++++++++++++++++++++++++++ process/errors.go | 3 ++ 3 files changed, 92 insertions(+), 10 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index a4ae5f124ee..471cc9f1f8d 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -2348,23 +2348,37 @@ func (bp *baseProcessor) getIndexOfFirstMiniBlockToBeExecuted(header data.Header return 0, nil } - for index, miniBlockHeaderHandler := range header.GetMiniBlockHeaderHandlers() { - if miniBlockHeaderHandler.GetProcessingType() == int32(block.Processed) { - if !bp.scheduledTxsExecutionHandler.IsMiniBlockExecuted(miniBlockHeaderHandler.GetHash()) { - return 0, fmt.Errorf("%w: mini block %s not executed", - process.ErrMiniBlockNotExecuted, + miniBlockHeaderHandlers := header.GetMiniBlockHeaderHandlers() + indexOfFirstMiniBlockToBeExecuted := len(miniBlockHeaderHandlers) + foundFirstNonProcessed := false + for index, miniBlockHeaderHandler := range miniBlockHeaderHandlers { + isProcessed := miniBlockHeaderHandler.GetProcessingType() == int32(block.Processed) + + // processed mini blocks are the ones executed as scheduled in the previous block and + // must form the contiguous leading prefix of the body; any later one is unverified + if foundFirstNonProcessed { + if isProcessed { + return 0, fmt.Errorf("%w: %s", + process.ErrProcessedMiniBlockNotInLeadingPrefix, hex.EncodeToString(miniBlockHeaderHandler.GetHash())) } - log.Debug("baseProcessor.getIndexOfFirstMiniBlockToBeExecuted: mini block is already executed", - "mb hash", miniBlockHeaderHandler.GetHash(), - "mb index", index) continue } - return index, nil + if !isProcessed { + indexOfFirstMiniBlockToBeExecuted = index + foundFirstNonProcessed = true + continue + } + + if !bp.scheduledTxsExecutionHandler.IsMiniBlockExecuted(miniBlockHeaderHandler.GetHash()) { + return 0, fmt.Errorf("%w: mini block %s not executed", + process.ErrMiniBlockNotExecuted, + hex.EncodeToString(miniBlockHeaderHandler.GetHash())) + } } - return len(header.GetMiniBlockHeaderHandlers()), nil + return indexOfFirstMiniBlockToBeExecuted, nil } func displayCleanupErrorMessage(message string, shardID uint32, noncesToPrevFinal uint64, err error) { diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index 52b40055af9..d1bb067fd84 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -2595,6 +2595,63 @@ func TestBaseProcessor_getIndexOfFirstMiniBlockToBeExecuted(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 1, index) }) + + t.Run("leading processed miniBlock not executed locally is rejected", func(t *testing.T) { + t.Parallel() + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + coreComponents.EnableEpochsHandlerField = enableEpochsHandlerMock.NewEnableEpochsHandlerStub(common.ScheduledMiniBlocksFlag) + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + arguments.ScheduledTxsExecutionHandler = &testscommon.ScheduledTxsExecutionStub{ + IsMiniBlockExecutedCalled: func(_ []byte) bool { + return false + }, + } + bp, _ := blproc.NewShardProcessor(arguments) + + mbh := block.MiniBlockHeader{} + mbhReserved := block.MiniBlockHeaderReserved{ExecutionType: block.Processed} + mbh.Reserved, _ = mbhReserved.Marshal() + + metaBlock := &block.MetaBlock{MiniBlockHeaders: []block.MiniBlockHeader{mbh}} + + index, err := bp.GetIndexOfFirstMiniBlockToBeExecuted(metaBlock) + assert.Zero(t, index) + assert.ErrorIs(t, err, process.ErrMiniBlockNotExecuted) + }) + + t.Run("processed miniBlock after a non-processed one is rejected", func(t *testing.T) { + t.Parallel() + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + coreComponents.EnableEpochsHandlerField = enableEpochsHandlerMock.NewEnableEpochsHandlerStub(common.ScheduledMiniBlocksFlag) + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + arguments.ScheduledTxsExecutionHandler = &testscommon.ScheduledTxsExecutionStub{ + IsMiniBlockExecutedCalled: func(_ []byte) bool { + return true + }, + } + bp, _ := blproc.NewShardProcessor(arguments) + + mbhNormal := block.MiniBlockHeader{} + mbhNormalReserved := block.MiniBlockHeaderReserved{ExecutionType: block.Normal} + mbhNormal.Reserved, _ = mbhNormalReserved.Marshal() + + mbhProcessed := block.MiniBlockHeader{} + mbhProcessedReserved := block.MiniBlockHeaderReserved{ExecutionType: block.Processed} + mbhProcessed.Reserved, _ = mbhProcessedReserved.Marshal() + + metaBlock := &block.MetaBlock{ + MiniBlockHeaders: []block.MiniBlockHeader{ + mbhNormal, + mbhProcessed, + }, + } + + index, err := bp.GetIndexOfFirstMiniBlockToBeExecuted(metaBlock) + assert.Zero(t, index) + assert.ErrorIs(t, err, process.ErrProcessedMiniBlockNotInLeadingPrefix) + }) } func TestBaseProcessor_getFinalMiniBlocks(t *testing.T) { @@ -3211,6 +3268,14 @@ func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) assert.ErrorIs(t, err, process.ErrInvalidConstructionState) }) + t.Run("outgoing normal proposed with final index rejected", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, otherShard, block.TxBlock, false, 3) + mbh := makeMbh(mb, block.Normal, block.Proposed, 2) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrInvalidConstructionState) + }) + t.Run("non TxBlock cannot be scheduled", func(t *testing.T) { t.Parallel() mb := makeMb(blockShard, blockShard, block.SmartContractResultBlock, true, 2) diff --git a/process/errors.go b/process/errors.go index f712d3e4728..55b7cffdcf1 100644 --- a/process/errors.go +++ b/process/errors.go @@ -741,6 +741,9 @@ var ErrShardInfoOnEpochStartBlock = errors.New("epoch-start block should not con // ErrMiniBlockNotExecuted signals that a mini block was not executed locally var ErrMiniBlockNotExecuted = errors.New("mini block not executed") +// ErrProcessedMiniBlockNotInLeadingPrefix signals that a processed mini block was found outside the leading scheduled-executed prefix +var ErrProcessedMiniBlockNotInLeadingPrefix = errors.New("processed mini block found outside the leading scheduled-executed prefix") + // ErrNilRewardsHandler signals that rewards handler is nil var ErrNilRewardsHandler = errors.New("rewards handler is nil") From 17a53ef14f9fc6f3736fd03cf4de98d89770e710 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 20 May 2026 17:15:12 +0300 Subject: [PATCH 072/116] additional verification --- process/block/baseProcess.go | 4 ++++ process/block/baseProcess_test.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 471cc9f1f8d..4ad19d446df 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1048,6 +1048,10 @@ func checkConstructionStateProcessingTypeAndIndexesCorrectness( switch hdrPT { case int32(block.Normal): if senderIsBlockShard { + if bodyPT != int32(block.Normal) { + return fmt.Errorf("%w: Normal header at sender shard requires Normal body, got body=%d", + process.ErrProcessingTypeBodyHeaderMismatch, bodyPT) + } if constructionState != int32(block.Final) { return fmt.Errorf("%w: Normal header at sender shard requires Final, got %d", process.ErrInvalidConstructionState, constructionState) diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index d1bb067fd84..6a6f38c1243 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -3236,6 +3236,14 @@ func TestCheckConstructionStateProcessingTypeAndIndexesCorrectness(t *testing.T) assert.ErrorIs(t, err, process.ErrProcessingTypeBodyHeaderMismatch) }) + t.Run("sender shard normal header with scheduled body rejected", func(t *testing.T) { + t.Parallel() + mb := makeMb(blockShard, otherShard, block.TxBlock, true, 3) + mbh := makeMbh(mb, block.Normal, block.Final, 2) + err := blproc.CheckConstructionStateProcessingTypeAndIndexesCorrectness(mbh, mb, blockShard) + assert.ErrorIs(t, err, process.ErrProcessingTypeBodyHeaderMismatch) + }) + t.Run("processed must have sender equal block shard", func(t *testing.T) { t.Parallel() mb := makeMb(otherShard, blockShard, block.TxBlock, true, 3) From f20169b231e2291a49c46984323e6047ec795e50 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 21 May 2026 15:23:13 +0300 Subject: [PATCH 073/116] request proof if missing in trigger --- epochStart/interface.go | 1 + epochStart/shardchain/trigger.go | 8 +++ epochStart/shardchain/trigger_test.go | 73 +++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/epochStart/interface.go b/epochStart/interface.go index ee5c0ff01ed..8011d3a8dfd 100644 --- a/epochStart/interface.go +++ b/epochStart/interface.go @@ -64,6 +64,7 @@ type RequestHandler interface { GetNumPeersToQuery(key string) (int, int, error) RequestValidatorInfo(hash []byte) RequestValidatorsInfo(hashes [][]byte) + RequestEquivalentProofByHash(headerShard uint32, headerHash []byte) IsInterfaceNil() bool } diff --git a/epochStart/shardchain/trigger.go b/epochStart/shardchain/trigger.go index e6100f3ab3b..ebab0414fab 100644 --- a/epochStart/shardchain/trigger.go +++ b/epochStart/shardchain/trigger.go @@ -592,6 +592,14 @@ func (t *trigger) receivedMetaBlock(headerHandler data.HeaderHandler, metaBlockH if t.enableEpochsHandler.IsFlagEnabledInEpoch(common.AndromedaFlag, headerHandler.GetEpoch()) { proof, err := t.proofsPool.GetProof(headerHandler.GetShardID(), metaBlockHash) if err != nil { + metaHdr, ok := headerHandler.(data.MetaHeaderHandler) + if ok && metaHdr.IsStartOfEpochBlock() && metaHdr.GetEpoch() > t.Epoch() { + log.Debug("proof not found for epoch start meta header, requesting it", + "header hash", metaBlockHash, + "epoch", headerHandler.GetEpoch(), + ) + go t.requestHandler.RequestEquivalentProofByHash(core.MetachainShardId, metaBlockHash) + } return } diff --git a/epochStart/shardchain/trigger_test.go b/epochStart/shardchain/trigger_test.go index 4a5d2e40998..34dd10d97be 100644 --- a/epochStart/shardchain/trigger_test.go +++ b/epochStart/shardchain/trigger_test.go @@ -5,7 +5,10 @@ import ( "errors" "fmt" "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" @@ -15,6 +18,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/mock" @@ -766,6 +770,75 @@ func TestTrigger_ReceivedHeaderChangeEpochWithoutPrevHeader(t *testing.T) { require.True(t, epochStartTrigger.isEpochStart) } +func TestTrigger_ReceivedMetaBlock_WithoutProof(t *testing.T) { + t.Parallel() + + t.Run("receivedMetaBlock should request proof when missing", func(t *testing.T) { + t.Parallel() + + var proofRequested atomic.Int32 + var requestedHashMut sync.Mutex + var requestedHash []byte + + args := createMockShardEpochStartTriggerArguments() + args.Epoch = 5 + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestEquivalentProofByHashCalled: func(headerShard uint32, headerHash []byte) { + requestedHashMut.Lock() + requestedHash = headerHash + requestedHashMut.Unlock() + proofRequested.Add(1) + }, + } + + args.DataPool = &dataRetrieverMock.PoolsHolderStub{ + HeadersCalled: func() dataRetriever.HeadersPool { + return &mock.HeadersCacherStub{} + }, + MiniBlocksCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, + CurrEpochValidatorInfoCalled: func() dataRetriever.ValidatorInfoCacher { + return &vic.ValidatorInfoCacherStub{} + }, + ProofsCalled: func() dataRetriever.ProofsPool { + return &dataRetrieverMock.ProofsPoolMock{ + GetProofCalled: func(_ uint32, _ []byte) (data.HeaderProofHandler, error) { + return nil, errors.New("proof not found") + }, + } + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + metaBlockHash := []byte("metablock-hash") + et.receivedMetaBlock(&block.MetaBlock{ + Nonce: 10, + Round: 42, + Epoch: 6, + EpochStart: block.EpochStart{LastFinalizedHeaders: []block.EpochStartShardData{{}}}, + }, metaBlockHash) + + time.Sleep(10 * time.Millisecond) + + require.Equal(t, int32(1), proofRequested.Load()) + + requestedHashMut.Lock() + require.Equal(t, metaBlockHash, requestedHash) + requestedHashMut.Unlock() + }) +} + func TestTrigger_ClearMissingValidatorsInfoMapShouldWork(t *testing.T) { t.Parallel() From 7f486102cd2863a321e0c8a386db11610ba383b0 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 21 May 2026 17:00:58 +0300 Subject: [PATCH 074/116] return nil on already authenticated --- process/heartbeat/interceptedPeerAuthentication.go | 2 +- process/heartbeat/interceptedPeerAuthentication_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index f1b43d294a9..2d6cd363045 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -143,7 +143,7 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { // Early exit if mapping already exists existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { - return process.ErrPeerAlreadyAuthenticated + return nil } if existingInfo.AuthTimestamp > ipa.payload.Timestamp { diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 84ae30f5c48..7c77c0da5a0 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -290,7 +290,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { ipa, _ := NewInterceptedPeerAuthentication(arg) err := ipa.CheckValidity() - assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) + assert.NoError(t, err) }) t.Run("peer already authenticated with newer timestamp should return error", func(t *testing.T) { t.Parallel() From 8a2f382283ed5bfb1b8fe4b8db25ec4a3f03cb67 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 21 May 2026 17:07:14 +0300 Subject: [PATCH 075/116] fix comment --- process/heartbeat/interceptedPeerAuthentication_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 7c77c0da5a0..4db9b174821 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -292,7 +292,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.NoError(t, err) }) - t.Run("peer already authenticated with newer timestamp should return error", func(t *testing.T) { + t.Run("peer already authenticated with newer timestamp should early exit", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() From 545228d32b7948fdad64ab0bab41314562e4db6c Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Fri, 22 May 2026 11:47:32 +0300 Subject: [PATCH 076/116] proper checks on peer authentication interceptor --- epochStart/bootstrap/process.go | 2 +- factory/processing/processComponents.go | 2 +- .../node/heartbeatV2/heartbeatV2_test.go | 4 +- integrationTests/testHeartbeatNode.go | 2 +- integrationTests/testProcessorNode.go | 2 +- .../interceptedPeerAuthentication.go | 15 ++++- .../interceptedPeerAuthentication_test.go | 30 +++++++++- .../peerAuthenticationPayloadValidator.go | 20 ++----- ...peerAuthenticationPayloadValidator_test.go | 56 ++----------------- ...nterceptedPeerAuthenticationDataFactory.go | 5 +- ...AuthenticationInterceptorProcessor_test.go | 2 +- 11 files changed, 61 insertions(+), 79 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 4644742434f..93e7cd91346 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -1465,7 +1465,7 @@ func (e *epochStartBootstrap) createResolversContainer() error { storageService := disabled.NewChainStorer() - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(e.generalConfig.HeartbeatV2.HeartbeatExpiryTimespanInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator() if err != nil { return err } diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index be2ebe983af..9ae9dc1064b 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1395,7 +1395,7 @@ func (pcf *processComponentsFactory) newResolverContainerFactory() (dataRetrieve return disabledResolversContainer.NewDisabledResolversContainerFactory(), nil } - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator() if err != nil { return nil, err } diff --git a/integrationTests/node/heartbeatV2/heartbeatV2_test.go b/integrationTests/node/heartbeatV2/heartbeatV2_test.go index bd5c988617f..a74bd5bd8ac 100644 --- a/integrationTests/node/heartbeatV2/heartbeatV2_test.go +++ b/integrationTests/node/heartbeatV2/heartbeatV2_test.go @@ -194,10 +194,10 @@ func TestHeartbeatV2_PeerAuthenticationMessageExpiration(t *testing.T) { time.Sleep(time.Second * 5) - // first node should have not received the requested message because it is expired + // first node will receive the requested message even though it is expired lastPkBytes := requestHashes[len(requestHashes)-1] assert.False(t, nodes[0].RequestedItemsHandler.Has(string(lastPkBytes))) - assert.Equal(t, interactingNodes-2, nodes[0].DataPool.PeerAuthentications().Len()) + assert.Equal(t, interactingNodes-1, nodes[0].DataPool.PeerAuthentications().Len()) } func TestHeartbeatV2_AllPeersSendMessagesOnAllNetworks(t *testing.T) { diff --git a/integrationTests/testHeartbeatNode.go b/integrationTests/testHeartbeatNode.go index c407b8534a1..eb312d12cfb 100644 --- a/integrationTests/testHeartbeatNode.go +++ b/integrationTests/testHeartbeatNode.go @@ -527,7 +527,7 @@ func (thn *TestHeartbeatNode) initResolversAndRequesters() { _ = thn.MainMessenger.CreateTopic(common.ConsensusTopic+thn.ShardCoordinator.CommunicationIdentifier(thn.ShardCoordinator.SelfId()), true) - payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator(thn.heartbeatExpiryTimespanInSec) + payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator() resolverContainerFactoryArgs := resolverscontainer.FactoryArgs{ ShardCoordinator: thn.ShardCoordinator, MainMessenger: thn.MainMessenger, diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 3b763591c85..84402d80980 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -1511,7 +1511,7 @@ func (tpn *TestProcessorNode) initResolvers() { consensusTopic := common.ConsensusTopic + tpn.ShardCoordinator.CommunicationIdentifier(tpn.ShardCoordinator.SelfId()) _ = tpn.MainMessenger.CreateTopic(consensusTopic, true) _ = tpn.FullArchiveMessenger.CreateTopic(consensusTopic, true) - payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator(60) + payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator() preferredPeersHolder, _ := p2pFactory.NewPeersHolder([]string{}) fullArchivePreferredPeersHolder, _ := p2pFactory.NewPeersHolder([]string{}) diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 2d6cd363045..9d49ac5ef5d 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -22,6 +22,8 @@ type ArgInterceptedPeerAuthentication struct { PayloadValidator process.PeerAuthenticationPayloadValidator HardforkTriggerPubKey []byte PeerShardMapper process.PeerShardMapper + MessageOriginator core.PeerID + SelfPeerID core.PeerID } // interceptedPeerAuthentication is a wrapper over PeerAuthentication @@ -35,6 +37,8 @@ type interceptedPeerAuthentication struct { payloadValidator process.PeerAuthenticationPayloadValidator hardforkTriggerPubKey []byte peerShardMapper process.PeerShardMapper + messageOriginator core.PeerID + selfPeerID core.PeerID } // NewInterceptedPeerAuthentication tries to create a new intercepted peer authentication instance @@ -58,6 +62,8 @@ func NewInterceptedPeerAuthentication(arg ArgInterceptedPeerAuthentication) (*in payloadValidator: arg.PayloadValidator, hardforkTriggerPubKey: arg.HardforkTriggerPubKey, peerShardMapper: arg.PeerShardMapper, + messageOriginator: arg.MessageOriginator, + selfPeerID: arg.SelfPeerID, } intercepted.peerId = core.PeerID(intercepted.peerAuthentication.Pid) @@ -140,11 +146,16 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { return err } - // Early exit if mapping already exists + // Early exit if mapping already exists and the message is from itself existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) - if string(existingInfo.PkBytes) == string(ipa.Pubkey()) { + isFromSelf := ipa.messageOriginator == ipa.selfPeerID + pairExists := string(existingInfo.PkBytes) == string(ipa.Pubkey()) + if pairExists && isFromSelf { return nil } + if pairExists && !isFromSelf { + return process.ErrPeerAlreadyAuthenticated + } if existingInfo.AuthTimestamp > ipa.payload.Timestamp { return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthTimestamp) diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 4db9b174821..c6ce19ca8c2 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -268,7 +268,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err = ipa.CheckValidity() assert.True(t, errors.Is(err, expectedErr)) }) - t.Run("peer already authenticated with same pubkey should return error", func(t *testing.T) { + t.Run("peer already authenticated with same pubkey should early exit, message from self", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() @@ -292,7 +292,33 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.NoError(t, err) }) - t.Run("peer already authenticated with newer timestamp should early exit", func(t *testing.T) { + t.Run("peer already authenticated with same pubkey should return error, message not from self", func(t *testing.T) { + t.Parallel() + + providedPA := createDefaultInterceptedPeerAuthentication() + arg := createMockInterceptedPeerAuthenticationArg(providedPA) + arg.MessageOriginator = "originator" + arg.SelfPeerID = "self" + + arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ + VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { + require.Fail(t, "should have not been called") + return expectedErr + }, + } + arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + return core.P2PPeerInfo{ + PkBytes: providedPA.Pubkey, + } + }, + } + + ipa, _ := NewInterceptedPeerAuthentication(arg) + err := ipa.CheckValidity() + assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) + }) + t.Run("peer already authenticated with newer timestamp should return error", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() diff --git a/process/heartbeat/validator/peerAuthenticationPayloadValidator.go b/process/heartbeat/validator/peerAuthenticationPayloadValidator.go index c2cb7d7f90a..cb220a2e482 100644 --- a/process/heartbeat/validator/peerAuthenticationPayloadValidator.go +++ b/process/heartbeat/validator/peerAuthenticationPayloadValidator.go @@ -8,36 +8,28 @@ import ( ) const ( - minDurationInSec = 10 payloadExpiryThresholdInSec = 10 ) type peerAuthenticationPayloadValidator struct { - expiryTimespanInSec int64 - getTimeHandler func() time.Time + getTimeHandler func() time.Time } // NewPeerAuthenticationPayloadValidator creates a new peer authentication payload validator instance -func NewPeerAuthenticationPayloadValidator(expiryTimespanInSec int64) (*peerAuthenticationPayloadValidator, error) { - if expiryTimespanInSec < minDurationInSec { - return nil, process.ErrInvalidExpiryTimespan - } - +func NewPeerAuthenticationPayloadValidator() (*peerAuthenticationPayloadValidator, error) { return &peerAuthenticationPayloadValidator{ - expiryTimespanInSec: expiryTimespanInSec, - getTimeHandler: time.Now, + getTimeHandler: time.Now, }, nil } // ValidateTimestamp will return an error if the provided payload timestamp is not valid func (validator *peerAuthenticationPayloadValidator) ValidateTimestamp(payloadTimestamp int64) error { currentTimeStamp := validator.getTimeHandler().Unix() - minTimestampAllowed := currentTimeStamp - validator.expiryTimespanInSec maxTimestampAllowed := currentTimeStamp + payloadExpiryThresholdInSec - if payloadTimestamp < minTimestampAllowed || payloadTimestamp > maxTimestampAllowed { - return fmt.Errorf("%w message time stamp: %v, minimum: %v, maximum: %v", - process.ErrMessageExpired, payloadTimestamp, minTimestampAllowed, maxTimestampAllowed) + if payloadTimestamp > maxTimestampAllowed { + return fmt.Errorf("%w message time stamp: %v, maximum: %v", + process.ErrMessageExpired, payloadTimestamp, maxTimestampAllowed) } return nil diff --git a/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go b/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go index 1d924908012..3592127a278 100644 --- a/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go +++ b/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go @@ -10,33 +10,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestNewPeerAuthenticationPayloadValidator(t *testing.T) { - t.Parallel() - - t.Run("invalid expiry duration should error", func(t *testing.T) { - t.Parallel() - - valsToTest := int64(100) - - for i := int64(1); i <= valsToTest; i++ { - validator, err := NewPeerAuthenticationPayloadValidator(minDurationInSec - i) - assert.True(t, check.IfNil(validator)) - assert.Equal(t, process.ErrInvalidExpiryTimespan, err) - } - }) - t.Run("should work", func(t *testing.T) { - t.Parallel() - - valsToTest := int64(100) - - for i := int64(0); i < valsToTest; i++ { - validator, err := NewPeerAuthenticationPayloadValidator(minDurationInSec + i) - assert.False(t, check.IfNil(validator)) - assert.Nil(t, err) - } - }) -} - func TestPeerAuthenticationPayloadValidator_ValidateTimestamp(t *testing.T) { t.Parallel() @@ -44,36 +17,15 @@ func TestPeerAuthenticationPayloadValidator_ValidateTimestamp(t *testing.T) { t.Parallel() currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) + validator, _ := NewPeerAuthenticationPayloadValidator() + assert.False(t, check.IfNil(validator)) assert.Nil(t, validator.ValidateTimestamp(currentTime.Unix())) }) - t.Run("payload time stamp is exactly the minim accepted", func(t *testing.T) { - t.Parallel() - - currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) - validator.getTimeHandler = func() time.Time { - return currentTime.Add(time.Second * 1120) - } - minimumAccepted := currentTime.Add(time.Second * (1120 - minDurationInSec)) - assert.Nil(t, validator.ValidateTimestamp(minimumAccepted.Unix())) - }) - t.Run("payload time stamp is less than minim accepted", func(t *testing.T) { - t.Parallel() - - currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) - validator.getTimeHandler = func() time.Time { - return currentTime.Add(time.Second * 1120) - } - minimumAccepted := currentTime.Add(time.Second * (1120 - minDurationInSec - 1)) - assert.True(t, errors.Is(validator.ValidateTimestamp(minimumAccepted.Unix()), process.ErrMessageExpired)) - }) t.Run("payload time stamp is exactly the maximum accepted", func(t *testing.T) { t.Parallel() currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) + validator, _ := NewPeerAuthenticationPayloadValidator() validator.getTimeHandler = func() time.Time { return currentTime.Add(time.Second * 1120) } @@ -84,7 +36,7 @@ func TestPeerAuthenticationPayloadValidator_ValidateTimestamp(t *testing.T) { t.Parallel() currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) + validator, _ := NewPeerAuthenticationPayloadValidator() validator.getTimeHandler = func() time.Time { return currentTime.Add(time.Second * 1120) } diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index a425dc3233a..b3dfb78c434 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -31,7 +31,7 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) return nil, err } - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(arg.HeartbeatExpiryTimespanInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator() if err != nil { return nil, err } @@ -74,7 +74,7 @@ func checkArgInterceptedDataFactory(args ArgInterceptedDataFactory) error { } // Create creates instances of InterceptedData by unmarshalling provided buffer -func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ core.PeerID) (process.InterceptedData, error) { +func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, messageOriginator core.PeerID) (process.InterceptedData, error) { arg := heartbeat.ArgInterceptedPeerAuthentication{ ArgBaseInterceptedHeartbeat: heartbeat.ArgBaseInterceptedHeartbeat{ DataBuff: buff, @@ -86,6 +86,7 @@ func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ cor PayloadValidator: ipadf.payloadValidator, HardforkTriggerPubKey: ipadf.hardforkTriggerPubKey, PeerShardMapper: ipadf.peerShardMapper, + MessageOriginator: messageOriginator, } return heartbeat.NewInterceptedPeerAuthentication(arg) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index d941cb79df1..865d3570463 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -51,7 +51,7 @@ func createInterceptedPeerAuthentication() *heartbeatMessages.PeerAuthentication } func createMockInterceptedPeerAuthentication() process.InterceptedData { - payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator(30) + payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator() arg := heartbeat.ArgInterceptedPeerAuthentication{ ArgBaseInterceptedHeartbeat: heartbeat.ArgBaseInterceptedHeartbeat{ From 9ec1abfcb7d290d2acb765589266c5685e287e61 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Fri, 22 May 2026 14:15:46 +0300 Subject: [PATCH 077/116] improve intercepted peer auth checks --- .../epochStartInterceptorsContainerFactory.go | 67 ++--- epochStart/bootstrap/process.go | 32 ++- epochStart/bootstrap/storageProcess.go | 30 +- epochStart/bootstrap/syncEpochStartMeta.go | 55 ++-- .../bootstrap/syncEpochStartMeta_test.go | 15 +- factory/processing/processComponents.go | 201 +++++++------- .../multiShard/hardFork/hardFork_test.go | 13 +- .../node/heartbeatV2/heartbeatV2_test.go | 4 +- integrationTests/testConsensusNode.go | 67 ++--- integrationTests/testFullNode.go | 67 ++--- integrationTests/testHeartbeatNode.go | 18 +- integrationTests/testProcessorNode.go | 136 ++++----- process/factory/interceptorscontainer/args.go | 67 ++--- .../metaInterceptorsContainerFactory.go | 34 +-- .../metaInterceptorsContainerFactory_test.go | 65 ++--- .../shardInterceptorsContainerFactory.go | 34 +-- .../shardInterceptorsContainerFactory_test.go | 65 ++--- .../interceptedPeerAuthentication.go | 83 +++--- .../interceptedPeerAuthentication_test.go | 65 +++-- .../peerAuthenticationPayloadValidator.go | 20 +- ...peerAuthenticationPayloadValidator_test.go | 56 +++- .../factory/argInterceptedDataFactory.go | 35 +-- ...interceptedEquivalentProofsFactory_test.go | 9 +- .../interceptedMetaHeaderDataFactory_test.go | 67 ++--- ...nterceptedPeerAuthenticationDataFactory.go | 52 ++-- ...AuthenticationInterceptorProcessor_test.go | 16 +- update/factory/exportHandlerFactory.go | 260 +++++++++--------- update/factory/fullSyncInterceptors.go | 74 ++--- 28 files changed, 918 insertions(+), 789 deletions(-) diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index a3fe40b3775..6f049aee27f 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -80,39 +80,40 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) hardforkTrigger := disabledFactory.HardforkTrigger() containerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: args.CoreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: nodesCoordinator, - MainMessenger: args.MainMessenger, - FullArchiveMessenger: args.FullArchiveMessenger, - Store: storer, - DataPool: args.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: feeHandler, - BlockBlackList: blackListHandler, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: args.WhiteListHandler, - WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - AntifloodHandler: antiFloodHandler, - ArgumentsParser: args.ArgumentsParser, - PreferredPeersHolder: disabled.NewPreferredPeersHolder(), - SizeCheckDelta: uint32(sizeCheckDelta), - RequestHandler: args.RequestHandler, - PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), - SignaturesHandler: args.SignaturesHandler, - HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MaxAllowedTrieNodeChunks: args.Config.Antiflood.MaxAllowedTrieNodeChunks, - TrieNodeChunksInactivityTimeout: time.Duration(args.Config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, - MainPeerShardMapper: peerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: args.NodeOperationMode, - InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + CoreComponents: args.CoreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: nodesCoordinator, + MainMessenger: args.MainMessenger, + FullArchiveMessenger: args.FullArchiveMessenger, + Store: storer, + DataPool: args.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: feeHandler, + BlockBlackList: blackListHandler, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: args.WhiteListHandler, + WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + AntifloodHandler: antiFloodHandler, + ArgumentsParser: args.ArgumentsParser, + PreferredPeersHolder: disabled.NewPreferredPeersHolder(), + SizeCheckDelta: uint32(sizeCheckDelta), + RequestHandler: args.RequestHandler, + PeerSignatureHandler: cryptoComponents.PeerSignatureHandler(), + SignaturesHandler: args.SignaturesHandler, + HeartbeatExpiryTimespanInSec: args.Config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + PeerAuthenticationTimeBetweenSendsInSec: args.Config.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec, + MaxAllowedTrieNodeChunks: args.Config.Antiflood.MaxAllowedTrieNodeChunks, + TrieNodeChunksInactivityTimeout: time.Duration(args.Config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, + MainPeerShardMapper: peerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: args.NodeOperationMode, + InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, } var interceptorsContainerFactory process.InterceptorsContainerFactory diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 93e7cd91346..8d06508dc7e 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -574,20 +574,22 @@ func (e *epochStartBootstrap) prepareComponentsToSyncFromNetwork() error { } argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{ - CoreComponentsHolder: e.coreComponentsHolder, - CryptoComponentsHolder: e.cryptoComponentsHolder, - RequestHandler: e.requestHandler, - Messenger: e.mainMessenger, - ShardCoordinator: e.shardCoordinator, - EconomicsData: e.economicsData, - WhitelistHandler: e.whiteListHandler, - StartInEpochConfig: epochStartConfig, - HeaderIntegrityVerifier: e.headerIntegrityVerifier, - MetaBlockProcessor: metaBlockProcessor, - InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, - ProofsPool: e.dataPool.Proofs(), - HeadersPool: e.dataPool.Headers(), - ProofsInterceptorProcessor: processor.NewEquivalentProofsInterceptorProcessor(), + CoreComponentsHolder: e.coreComponentsHolder, + CryptoComponentsHolder: e.cryptoComponentsHolder, + RequestHandler: e.requestHandler, + Messenger: e.mainMessenger, + ShardCoordinator: e.shardCoordinator, + EconomicsData: e.economicsData, + WhitelistHandler: e.whiteListHandler, + StartInEpochConfig: epochStartConfig, + HeaderIntegrityVerifier: e.headerIntegrityVerifier, + MetaBlockProcessor: metaBlockProcessor, + InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, + ProofsPool: e.dataPool.Proofs(), + HeadersPool: e.dataPool.Headers(), + ProofsInterceptorProcessor: processor.NewEquivalentProofsInterceptorProcessor(), + PeerAuthCacher: e.dataPool.PeerAuthentications(), + PeerAuthenticationTimeBetweenSendsInSec: e.generalConfig.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec, } e.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer) if err != nil { @@ -1465,7 +1467,7 @@ func (e *epochStartBootstrap) createResolversContainer() error { storageService := disabled.NewChainStorer() - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator() + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(e.generalConfig.HeartbeatV2.HeartbeatExpiryTimespanInSec) if err != nil { return err } diff --git a/epochStart/bootstrap/storageProcess.go b/epochStart/bootstrap/storageProcess.go index 307e9469bd2..90b9b98ffbd 100644 --- a/epochStart/bootstrap/storageProcess.go +++ b/epochStart/bootstrap/storageProcess.go @@ -179,20 +179,22 @@ func (sesb *storageEpochStartBootstrap) prepareComponentsToSync() error { } argsEpochStartSyncer := ArgsNewEpochStartMetaSyncer{ - CoreComponentsHolder: sesb.coreComponentsHolder, - CryptoComponentsHolder: sesb.cryptoComponentsHolder, - RequestHandler: sesb.requestHandler, - Messenger: sesb.mainMessenger, - ShardCoordinator: sesb.shardCoordinator, - EconomicsData: sesb.economicsData, - WhitelistHandler: sesb.whiteListHandler, - StartInEpochConfig: sesb.generalConfig.EpochStartConfig, - HeaderIntegrityVerifier: sesb.headerIntegrityVerifier, - MetaBlockProcessor: metablockProcessor, - InterceptedDataVerifierFactory: sesb.interceptedDataVerifierFactory, - ProofsPool: sesb.dataPool.Proofs(), - HeadersPool: sesb.dataPool.Headers(), - ProofsInterceptorProcessor: processor.NewEquivalentProofsInterceptorProcessor(), + CoreComponentsHolder: sesb.coreComponentsHolder, + CryptoComponentsHolder: sesb.cryptoComponentsHolder, + RequestHandler: sesb.requestHandler, + Messenger: sesb.mainMessenger, + ShardCoordinator: sesb.shardCoordinator, + EconomicsData: sesb.economicsData, + WhitelistHandler: sesb.whiteListHandler, + StartInEpochConfig: sesb.generalConfig.EpochStartConfig, + HeaderIntegrityVerifier: sesb.headerIntegrityVerifier, + MetaBlockProcessor: metablockProcessor, + InterceptedDataVerifierFactory: sesb.interceptedDataVerifierFactory, + ProofsPool: sesb.dataPool.Proofs(), + HeadersPool: sesb.dataPool.Headers(), + ProofsInterceptorProcessor: processor.NewEquivalentProofsInterceptorProcessor(), + PeerAuthCacher: sesb.dataPool.PeerAuthentications(), + PeerAuthenticationTimeBetweenSendsInSec: sesb.generalConfig.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec, } sesb.epochStartMetaBlockSyncer, err = NewEpochStartMetaSyncer(argsEpochStartSyncer) diff --git a/epochStart/bootstrap/syncEpochStartMeta.go b/epochStart/bootstrap/syncEpochStartMeta.go index 447a52b1924..e67fd98d377 100644 --- a/epochStart/bootstrap/syncEpochStartMeta.go +++ b/epochStart/bootstrap/syncEpochStartMeta.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" @@ -37,21 +38,23 @@ type epochStartMetaSyncer struct { // ArgsNewEpochStartMetaSyncer - type ArgsNewEpochStartMetaSyncer struct { - CoreComponentsHolder process.CoreComponentsHolder - CryptoComponentsHolder process.CryptoComponentsHolder - RequestHandler RequestHandler - Messenger Messenger - ShardCoordinator sharding.Coordinator - EconomicsData process.EconomicsDataHandler - WhitelistHandler process.WhiteListHandler - StartInEpochConfig config.EpochStartConfig - ArgsParser process.ArgumentsParser - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - MetaBlockProcessor EpochStartMetaBlockInterceptorProcessor - InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory - ProofsPool dataRetriever.ProofsPool - HeadersPool dataRetriever.HeadersPool - ProofsInterceptorProcessor process.InterceptorProcessor + CoreComponentsHolder process.CoreComponentsHolder + CryptoComponentsHolder process.CryptoComponentsHolder + RequestHandler RequestHandler + Messenger Messenger + ShardCoordinator sharding.Coordinator + EconomicsData process.EconomicsDataHandler + WhitelistHandler process.WhiteListHandler + StartInEpochConfig config.EpochStartConfig + ArgsParser process.ArgumentsParser + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + MetaBlockProcessor EpochStartMetaBlockInterceptorProcessor + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + ProofsPool dataRetriever.ProofsPool + HeadersPool dataRetriever.HeadersPool + ProofsInterceptorProcessor process.InterceptorProcessor + PeerAuthCacher storage.Cacher + PeerAuthenticationTimeBetweenSendsInSec int64 } // NewEpochStartMetaSyncer will return a new instance of epochStartMetaSyncer @@ -88,16 +91,18 @@ func NewEpochStartMetaSyncer(args ArgsNewEpochStartMetaSyncer) (*epochStartMetaS } argsInterceptedDataFactory := interceptorsFactory.ArgInterceptedDataFactory{ - CoreComponents: args.CoreComponentsHolder, - CryptoComponents: args.CryptoComponentsHolder, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: disabled.NewNodesCoordinator(), - FeeHandler: args.EconomicsData, - HeaderSigVerifier: disabled.NewHeaderSigVerifier(), - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - ValidityAttester: disabled.NewValidityAttester(), - EpochStartTrigger: disabled.NewEpochStartTrigger(), - ArgsParser: args.ArgsParser, + CoreComponents: args.CoreComponentsHolder, + CryptoComponents: args.CryptoComponentsHolder, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: disabled.NewNodesCoordinator(), + FeeHandler: args.EconomicsData, + HeaderSigVerifier: disabled.NewHeaderSigVerifier(), + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + ValidityAttester: disabled.NewValidityAttester(), + EpochStartTrigger: disabled.NewEpochStartTrigger(), + ArgsParser: args.ArgsParser, + PeerAuthCacher: args.PeerAuthCacher, + PeerAuthenticationTimeBetweenSendsInSec: args.PeerAuthenticationTimeBetweenSendsInSec, } argsInterceptedMetaHeaderFactory := interceptorsFactory.ArgInterceptedMetaHeaderFactory{ ArgInterceptedDataFactory: argsInterceptedDataFactory, diff --git a/epochStart/bootstrap/syncEpochStartMeta_test.go b/epochStart/bootstrap/syncEpochStartMeta_test.go index 54055e6c663..e856ed921ac 100644 --- a/epochStart/bootstrap/syncEpochStartMeta_test.go +++ b/epochStart/bootstrap/syncEpochStartMeta_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/pool" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -173,11 +174,13 @@ func getEpochStartSyncerArgs() ArgsNewEpochStartMetaSyncer { MinNumConnectedPeersToStart: 2, MinNumOfPeersToConsiderBlockValid: 2, }, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - MetaBlockProcessor: &mock.EpochStartMetaBlockProcessorStub{}, - InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{}, - ProofsPool: &dataRetriever.ProofsPoolMock{}, - HeadersPool: &pool.HeadersPoolStub{}, - ProofsInterceptorProcessor: &processMock.InterceptorProcessorStub{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + MetaBlockProcessor: &mock.EpochStartMetaBlockProcessorStub{}, + InterceptedDataVerifierFactory: &processMock.InterceptedDataVerifierFactoryMock{}, + ProofsPool: &dataRetriever.ProofsPoolMock{}, + HeadersPool: &pool.HeadersPoolStub{}, + ProofsInterceptorProcessor: &processMock.InterceptorProcessorStub{}, + PeerAuthCacher: cache.NewCacherStub(), + PeerAuthenticationTimeBetweenSendsInSec: 60, } } diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 9ae9dc1064b..066efb32d21 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1395,7 +1395,7 @@ func (pcf *processComponentsFactory) newResolverContainerFactory() (dataRetrieve return disabledResolversContainer.NewDisabledResolversContainerFactory(), nil } - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator() + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec) if err != nil { return nil, err } @@ -1698,39 +1698,40 @@ func (pcf *processComponentsFactory) newShardInterceptorContainerFactory( ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) shardInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - Accounts: pcf.state.AccountsAdapterAPI(), - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - NodesCoordinator: pcf.nodesCoordinator, - MainMessenger: pcf.network.NetworkMessenger(), - FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), - Store: pcf.data.StorageService(), - DataPool: pcf.data.Datapool(), - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: pcf.coreData.EconomicsData(), - BlockBlackList: headerBlackList, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: headerIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - AntifloodHandler: pcf.network.InputAntiFloodHandler(), - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), - SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, - RequestHandler: requestHandler, - PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), - SignaturesHandler: pcf.network.NetworkMessenger(), - HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, - TrieNodeChunksInactivityTimeout: time.Duration(pcf.config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, - MainPeerShardMapper: mainPeerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: nodeOperationMode, - InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + Accounts: pcf.state.AccountsAdapterAPI(), + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + NodesCoordinator: pcf.nodesCoordinator, + MainMessenger: pcf.network.NetworkMessenger(), + FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), + Store: pcf.data.StorageService(), + DataPool: pcf.data.Datapool(), + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: pcf.coreData.EconomicsData(), + BlockBlackList: headerBlackList, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: headerIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + AntifloodHandler: pcf.network.InputAntiFloodHandler(), + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), + SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, + RequestHandler: requestHandler, + PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), + SignaturesHandler: pcf.network.NetworkMessenger(), + HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + PeerAuthenticationTimeBetweenSendsInSec: pcf.config.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec, + MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, + TrieNodeChunksInactivityTimeout: time.Duration(pcf.config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, + MainPeerShardMapper: mainPeerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } interceptorContainerFactory, err := interceptorscontainer.NewShardInterceptorsContainerFactory(shardInterceptorsContainerFactoryArgs) @@ -1754,39 +1755,40 @@ func (pcf *processComponentsFactory) newMetaInterceptorContainerFactory( ) (process.InterceptorsContainerFactory, process.TimeCacher, error) { headerBlackList := cache.NewTimeCache(timeSpanForBadHeaders) metaInterceptorsContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - NodesCoordinator: pcf.nodesCoordinator, - MainMessenger: pcf.network.NetworkMessenger(), - FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), - Store: pcf.data.StorageService(), - DataPool: pcf.data.Datapool(), - Accounts: pcf.state.AccountsAdapterAPI(), - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: pcf.coreData.EconomicsData(), - BlockBlackList: headerBlackList, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: headerIntegrityVerifier, - ValidityAttester: validityAttester, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - AntifloodHandler: pcf.network.InputAntiFloodHandler(), - ArgumentsParser: smartContract.NewArgumentParser(), - SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, - PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), - RequestHandler: requestHandler, - PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), - SignaturesHandler: pcf.network.NetworkMessenger(), - HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, - MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, - TrieNodeChunksInactivityTimeout: time.Duration(pcf.config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, - MainPeerShardMapper: mainPeerShardMapper, - FullArchivePeerShardMapper: fullArchivePeerShardMapper, - HardforkTrigger: hardforkTrigger, - NodeOperationMode: nodeOperationMode, - InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + NodesCoordinator: pcf.nodesCoordinator, + MainMessenger: pcf.network.NetworkMessenger(), + FullArchiveMessenger: pcf.network.FullArchiveNetworkMessenger(), + Store: pcf.data.StorageService(), + DataPool: pcf.data.Datapool(), + Accounts: pcf.state.AccountsAdapterAPI(), + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: pcf.coreData.EconomicsData(), + BlockBlackList: headerBlackList, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: headerIntegrityVerifier, + ValidityAttester: validityAttester, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + AntifloodHandler: pcf.network.InputAntiFloodHandler(), + ArgumentsParser: smartContract.NewArgumentParser(), + SizeCheckDelta: pcf.config.Marshalizer.SizeCheckDelta, + PreferredPeersHolder: pcf.network.PreferredPeersHolderHandler(), + RequestHandler: requestHandler, + PeerSignatureHandler: pcf.crypto.PeerSignatureHandler(), + SignaturesHandler: pcf.network.NetworkMessenger(), + HeartbeatExpiryTimespanInSec: pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec, + PeerAuthenticationTimeBetweenSendsInSec: pcf.config.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec, + MaxAllowedTrieNodeChunks: pcf.config.Antiflood.MaxAllowedTrieNodeChunks, + TrieNodeChunksInactivityTimeout: time.Duration(pcf.config.Antiflood.TrieNodeChunksInactivityTimeoutInSec) * time.Second, + MainPeerShardMapper: mainPeerShardMapper, + FullArchivePeerShardMapper: fullArchivePeerShardMapper, + HardforkTrigger: hardforkTrigger, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, } interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorsContainerFactoryArgs) @@ -1867,38 +1869,39 @@ func (pcf *processComponentsFactory) createExportFactoryHandler( nodeOperationMode = common.FullArchiveMode } argsExporter := updateFactory.ArgsExporter{ - CoreComponents: pcf.coreData, - CryptoComponents: pcf.crypto, - StatusCoreComponents: pcf.statusCoreComponents, - NetworkComponents: pcf.network, - HeaderValidator: headerValidator, - DataPool: pcf.data.Datapool(), - StorageService: pcf.data.StorageService(), - RequestHandler: requestHandler, - ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), - ActiveAccountsDBs: accountsDBs, - ExistingResolvers: resolversContainer, - ExistingRequesters: requestersContainer, - ExportFolder: exportFolder, - ExportTriesStorageConfig: hardforkConfig.ExportTriesStorageConfig, - ExportStateStorageConfig: hardforkConfig.ExportStateStorageConfig, - ExportStateKeysConfig: hardforkConfig.ExportKeysStorageConfig, - MaxTrieLevelInMemory: pcf.config.StateTriesConfig.MaxStateTrieLevelInMemory, - WhiteListHandler: pcf.whiteListHandler, - WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, - MainInterceptorsContainer: mainInterceptorsContainer, - FullArchiveInterceptorsContainer: fullArchiveInterceptorsContainer, - NodesCoordinator: pcf.nodesCoordinator, - HeaderSigVerifier: headerSigVerifier, - HeaderIntegrityVerifier: pcf.bootstrapComponents.HeaderIntegrityVerifier(), - ValidityAttester: blockTracker, - RoundHandler: pcf.coreData.RoundHandler(), - InterceptorDebugConfig: pcf.config.Debug.InterceptorResolver, - MaxHardCapForMissingNodes: pcf.config.TrieSync.MaxHardCapForMissingNodes, - NumConcurrentTrieSyncers: pcf.config.TrieSync.NumConcurrentTrieSyncers, - TrieSyncerVersion: pcf.config.TrieSync.TrieSyncerVersion, - NodeOperationMode: nodeOperationMode, - InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, + CoreComponents: pcf.coreData, + CryptoComponents: pcf.crypto, + StatusCoreComponents: pcf.statusCoreComponents, + NetworkComponents: pcf.network, + HeaderValidator: headerValidator, + DataPool: pcf.data.Datapool(), + StorageService: pcf.data.StorageService(), + RequestHandler: requestHandler, + ShardCoordinator: pcf.bootstrapComponents.ShardCoordinator(), + ActiveAccountsDBs: accountsDBs, + ExistingResolvers: resolversContainer, + ExistingRequesters: requestersContainer, + ExportFolder: exportFolder, + ExportTriesStorageConfig: hardforkConfig.ExportTriesStorageConfig, + ExportStateStorageConfig: hardforkConfig.ExportStateStorageConfig, + ExportStateKeysConfig: hardforkConfig.ExportKeysStorageConfig, + MaxTrieLevelInMemory: pcf.config.StateTriesConfig.MaxStateTrieLevelInMemory, + WhiteListHandler: pcf.whiteListHandler, + WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, + MainInterceptorsContainer: mainInterceptorsContainer, + FullArchiveInterceptorsContainer: fullArchiveInterceptorsContainer, + NodesCoordinator: pcf.nodesCoordinator, + HeaderSigVerifier: headerSigVerifier, + HeaderIntegrityVerifier: pcf.bootstrapComponents.HeaderIntegrityVerifier(), + ValidityAttester: blockTracker, + RoundHandler: pcf.coreData.RoundHandler(), + InterceptorDebugConfig: pcf.config.Debug.InterceptorResolver, + MaxHardCapForMissingNodes: pcf.config.TrieSync.MaxHardCapForMissingNodes, + NumConcurrentTrieSyncers: pcf.config.TrieSync.NumConcurrentTrieSyncers, + TrieSyncerVersion: pcf.config.TrieSync.TrieSyncerVersion, + NodeOperationMode: nodeOperationMode, + InterceptedDataVerifierFactory: pcf.interceptedDataVerifierFactory, + PeerAuthenticationTimeBetweenSendsInSec: pcf.config.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec, } return updateFactory.NewExportHandlerFactory(argsExporter) } diff --git a/integrationTests/multiShard/hardFork/hardFork_test.go b/integrationTests/multiShard/hardFork/hardFork_test.go index 5b2754110ef..b0ffe3efd22 100644 --- a/integrationTests/multiShard/hardFork/hardFork_test.go +++ b/integrationTests/multiShard/hardFork/hardFork_test.go @@ -663,12 +663,13 @@ func createHardForkExporter( NumResolveFailureThreshold: 3, DebugLineExpiration: 3, }, - MaxHardCapForMissingNodes: 500, - NumConcurrentTrieSyncers: 50, - TrieSyncerVersion: 2, - CheckNodesOnDisk: false, - NodeOperationMode: node.NodeOperationMode, - InterceptedDataVerifierFactory: interceptorFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierFactoryArgs), + MaxHardCapForMissingNodes: 500, + NumConcurrentTrieSyncers: 50, + TrieSyncerVersion: 2, + CheckNodesOnDisk: false, + NodeOperationMode: node.NodeOperationMode, + InterceptedDataVerifierFactory: interceptorFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierFactoryArgs), + PeerAuthenticationTimeBetweenSendsInSec: 60, } exportHandler, err := factory.NewExportHandlerFactory(argsExportHandler) diff --git a/integrationTests/node/heartbeatV2/heartbeatV2_test.go b/integrationTests/node/heartbeatV2/heartbeatV2_test.go index a74bd5bd8ac..bd5c988617f 100644 --- a/integrationTests/node/heartbeatV2/heartbeatV2_test.go +++ b/integrationTests/node/heartbeatV2/heartbeatV2_test.go @@ -194,10 +194,10 @@ func TestHeartbeatV2_PeerAuthenticationMessageExpiration(t *testing.T) { time.Sleep(time.Second * 5) - // first node will receive the requested message even though it is expired + // first node should have not received the requested message because it is expired lastPkBytes := requestHashes[len(requestHashes)-1] assert.False(t, nodes[0].RequestedItemsHandler.Has(string(lastPkBytes))) - assert.Equal(t, interactingNodes-1, nodes[0].DataPool.PeerAuthentications().Len()) + assert.Equal(t, interactingNodes-2, nodes[0].DataPool.PeerAuthentications().Len()) } func TestHeartbeatV2_AllPeersSendMessagesOnAllNetworks(t *testing.T) { diff --git a/integrationTests/testConsensusNode.go b/integrationTests/testConsensusNode.go index d24ef3786c3..cbe6311813f 100644 --- a/integrationTests/testConsensusNode.go +++ b/integrationTests/testConsensusNode.go @@ -464,39 +464,40 @@ func (tcn *TestConsensusNode) initInterceptors( whiteListerVerifiedTxs, _ := interceptors.NewWhiteListDataVerifier(cacheVerified) interceptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: tcn.ShardCoordinator, - NodesCoordinator: tcn.NodesCoordinator, - MainMessenger: tcn.MainMessenger, - FullArchiveMessenger: tcn.FullArchiveMessenger, - Store: storage, - DataPool: tcn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: blockBlackListHandler, - HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), - ValidityAttester: blockTracker, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: whiteLstHandler, - WhiteListerVerifiedTxs: whiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second, - MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), - FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: tcn.ShardCoordinator, + NodesCoordinator: tcn.NodesCoordinator, + MainMessenger: tcn.MainMessenger, + FullArchiveMessenger: tcn.FullArchiveMessenger, + Store: storage, + DataPool: tcn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: blockBlackListHandler, + HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), + ValidityAttester: blockTracker, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: whiteLstHandler, + WhiteListerVerifiedTxs: whiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerAuthenticationTimeBetweenSendsInSec: 60, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), + FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } if tcn.ShardCoordinator.SelfId() == core.MetachainShardId { interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 22bddfc3aee..5dd391cb53f 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -720,39 +720,40 @@ func (tcn *TestFullNode) initInterceptors( whiteListerVerifiedTxs, _ := interceptors.NewWhiteListDataVerifier(cacheVerified) interceptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: accountsAdapter, - ShardCoordinator: tcn.ShardCoordinator, - NodesCoordinator: tcn.NodesCoordinator, - MainMessenger: tcn.MainMessenger, - FullArchiveMessenger: tcn.FullArchiveMessenger, - Store: storage, - DataPool: tcn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: blockBlackListHandler, - HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), - ValidityAttester: blockTracker, - EpochStartTrigger: epochStartTrigger, - WhiteListHandler: whiteLstHandler, - WhiteListerVerifiedTxs: whiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second, - MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), - FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: accountsAdapter, + ShardCoordinator: tcn.ShardCoordinator, + NodesCoordinator: tcn.NodesCoordinator, + MainMessenger: tcn.MainMessenger, + FullArchiveMessenger: tcn.FullArchiveMessenger, + Store: storage, + DataPool: tcn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: blockBlackListHandler, + HeaderSigVerifier: &consensusMocks.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: CreateHeaderIntegrityVerifier(), + ValidityAttester: blockTracker, + EpochStartTrigger: epochStartTrigger, + WhiteListHandler: whiteLstHandler, + WhiteListerVerifiedTxs: whiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerAuthenticationTimeBetweenSendsInSec: 60, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), + FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } if tcn.ShardCoordinator.SelfId() == core.MetachainShardId { interceptorContainerFactory, err := interceptorscontainer.NewMetaInterceptorsContainerFactory(interceptorContainerFactoryArgs) diff --git a/integrationTests/testHeartbeatNode.go b/integrationTests/testHeartbeatNode.go index eb312d12cfb..1936b2590d0 100644 --- a/integrationTests/testHeartbeatNode.go +++ b/integrationTests/testHeartbeatNode.go @@ -527,7 +527,7 @@ func (thn *TestHeartbeatNode) initResolversAndRequesters() { _ = thn.MainMessenger.CreateTopic(common.ConsensusTopic+thn.ShardCoordinator.CommunicationIdentifier(thn.ShardCoordinator.SelfId()), true) - payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator() + payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator(thn.heartbeatExpiryTimespanInSec) resolverContainerFactoryArgs := resolverscontainer.FactoryArgs{ ShardCoordinator: thn.ShardCoordinator, MainMessenger: thn.MainMessenger, @@ -640,13 +640,15 @@ func (thn *TestHeartbeatNode) initInterceptors() { IntMarsh: TestMarshaller, HardforkTriggerPubKeyField: []byte(providedHardforkPubKey), }, - ShardCoordinator: thn.ShardCoordinator, - NodesCoordinator: thn.NodesCoordinator, - PeerSignatureHandler: thn.PeerSigHandler, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: thn.heartbeatExpiryTimespanInSec, - PeerID: thn.MainMessenger.ID(), - PeerShardMapper: thn.MainPeerShardMapper, + ShardCoordinator: thn.ShardCoordinator, + NodesCoordinator: thn.NodesCoordinator, + PeerSignatureHandler: thn.PeerSigHandler, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: thn.heartbeatExpiryTimespanInSec, + PeerID: thn.MainMessenger.ID(), + PeerShardMapper: thn.MainPeerShardMapper, + PeerAuthCacher: thn.DataPool.PeerAuthentications(), + PeerAuthenticationTimeBetweenSendsInSec: thn.heartbeatExpiryTimespanInSec, } thn.createPeerAuthInterceptor(argsFactory) diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 84402d80980..ea84784dce1 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -1362,39 +1362,40 @@ func (tpn *TestProcessorNode) initInterceptors(heartbeatPk string) { coreComponents.HardforkTriggerPubKeyField = providedHardforkPk metaInterceptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: tpn.AccntState, - ShardCoordinator: tpn.ShardCoordinator, - NodesCoordinator: tpn.NodesCoordinator, - MainMessenger: tpn.MainMessenger, - FullArchiveMessenger: tpn.FullArchiveMessenger, - Store: tpn.Storage, - DataPool: tpn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: tpn.EconomicsData, - BlockBlackList: tpn.BlockBlackListHandler, - HeaderSigVerifier: tpn.HeaderSigVerifier, - HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, - ValidityAttester: tpn.BlockTracker, - EpochStartTrigger: tpn.EpochStartTrigger, - WhiteListHandler: tpn.WhiteListHandler, - WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: tpn.RequestHandler, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second, - MainPeerShardMapper: tpn.MainPeerShardMapper, - FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, - HardforkTrigger: tpn.HardforkTrigger, - NodeOperationMode: tpn.NodeOperationMode, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: tpn.AccntState, + ShardCoordinator: tpn.ShardCoordinator, + NodesCoordinator: tpn.NodesCoordinator, + MainMessenger: tpn.MainMessenger, + FullArchiveMessenger: tpn.FullArchiveMessenger, + Store: tpn.Storage, + DataPool: tpn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: tpn.EconomicsData, + BlockBlackList: tpn.BlockBlackListHandler, + HeaderSigVerifier: tpn.HeaderSigVerifier, + HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, + ValidityAttester: tpn.BlockTracker, + EpochStartTrigger: tpn.EpochStartTrigger, + WhiteListHandler: tpn.WhiteListHandler, + WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: tpn.RequestHandler, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerAuthenticationTimeBetweenSendsInSec: 60, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: tpn.MainPeerShardMapper, + FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, + HardforkTrigger: tpn.HardforkTrigger, + NodeOperationMode: tpn.NodeOperationMode, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } interceptorContainerFactory, _ := interceptorscontainer.NewMetaInterceptorsContainerFactory(metaInterceptorContainerFactoryArgs) @@ -1433,39 +1434,40 @@ func (tpn *TestProcessorNode) initInterceptors(heartbeatPk string) { coreComponents.HardforkTriggerPubKeyField = providedHardforkPk shardIntereptorContainerFactoryArgs := interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - Accounts: tpn.AccntState, - ShardCoordinator: tpn.ShardCoordinator, - NodesCoordinator: tpn.NodesCoordinator, - MainMessenger: tpn.MainMessenger, - FullArchiveMessenger: tpn.FullArchiveMessenger, - Store: tpn.Storage, - DataPool: tpn.DataPool, - MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, - TxFeeHandler: tpn.EconomicsData, - BlockBlackList: tpn.BlockBlackListHandler, - HeaderSigVerifier: tpn.HeaderSigVerifier, - HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, - ValidityAttester: tpn.BlockTracker, - EpochStartTrigger: tpn.EpochStartTrigger, - WhiteListHandler: tpn.WhiteListHandler, - WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, - AntifloodHandler: &mock.NilAntifloodHandler{}, - ArgumentsParser: smartContract.NewArgumentParser(), - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - SizeCheckDelta: sizeCheckDelta, - RequestHandler: tpn.RequestHandler, - PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second, - MainPeerShardMapper: tpn.MainPeerShardMapper, - FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, - HardforkTrigger: tpn.HardforkTrigger, - NodeOperationMode: tpn.NodeOperationMode, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + Accounts: tpn.AccntState, + ShardCoordinator: tpn.ShardCoordinator, + NodesCoordinator: tpn.NodesCoordinator, + MainMessenger: tpn.MainMessenger, + FullArchiveMessenger: tpn.FullArchiveMessenger, + Store: tpn.Storage, + DataPool: tpn.DataPool, + MaxTxNonceDeltaAllowed: common.MaxTxNonceDeltaAllowed, + TxFeeHandler: tpn.EconomicsData, + BlockBlackList: tpn.BlockBlackListHandler, + HeaderSigVerifier: tpn.HeaderSigVerifier, + HeaderIntegrityVerifier: tpn.HeaderIntegrityVerifier, + ValidityAttester: tpn.BlockTracker, + EpochStartTrigger: tpn.EpochStartTrigger, + WhiteListHandler: tpn.WhiteListHandler, + WhiteListerVerifiedTxs: tpn.WhiteListerVerifiedTxs, + AntifloodHandler: &mock.NilAntifloodHandler{}, + ArgumentsParser: smartContract.NewArgumentParser(), + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + SizeCheckDelta: sizeCheckDelta, + RequestHandler: tpn.RequestHandler, + PeerSignatureHandler: &processMock.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerAuthenticationTimeBetweenSendsInSec: 60, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: tpn.MainPeerShardMapper, + FullArchivePeerShardMapper: tpn.FullArchivePeerShardMapper, + HardforkTrigger: tpn.HardforkTrigger, + NodeOperationMode: tpn.NodeOperationMode, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), } interceptorContainerFactory, _ := interceptorscontainer.NewShardInterceptorsContainerFactory(shardIntereptorContainerFactoryArgs) @@ -1511,7 +1513,7 @@ func (tpn *TestProcessorNode) initResolvers() { consensusTopic := common.ConsensusTopic + tpn.ShardCoordinator.CommunicationIdentifier(tpn.ShardCoordinator.SelfId()) _ = tpn.MainMessenger.CreateTopic(consensusTopic, true) _ = tpn.FullArchiveMessenger.CreateTopic(consensusTopic, true) - payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator() + payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator(60) preferredPeersHolder, _ := p2pFactory.NewPeersHolder([]string{}) fullArchivePreferredPeersHolder, _ := p2pFactory.NewPeersHolder([]string{}) diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index 6a2832dd8ca..2c4add64223 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -16,37 +16,38 @@ import ( // CommonInterceptorsContainerFactoryArgs holds the arguments needed for the metachain/shard interceptors factories type CommonInterceptorsContainerFactoryArgs struct { - CoreComponents process.CoreComponentsHolder - CryptoComponents process.CryptoComponentsHolder - Accounts state.AccountsAdapter - ShardCoordinator sharding.Coordinator - NodesCoordinator nodesCoordinator.NodesCoordinator - MainMessenger process.TopicHandler - FullArchiveMessenger process.TopicHandler - Store dataRetriever.StorageService - DataPool dataRetriever.PoolsHolder - MaxTxNonceDeltaAllowed int - TxFeeHandler process.FeeHandler - BlockBlackList process.TimeCacher - HeaderSigVerifier process.InterceptedHeaderSigVerifier - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - ValidityAttester process.ValidityAttester - EpochStartTrigger process.EpochStartTriggerHandler - WhiteListHandler process.WhiteListHandler - WhiteListerVerifiedTxs process.WhiteListHandler - AntifloodHandler process.P2PAntifloodHandler - ArgumentsParser process.ArgumentsParser - PreferredPeersHolder process.PreferredPeersHolderHandler - SizeCheckDelta uint32 - RequestHandler process.RequestHandler - PeerSignatureHandler crypto.PeerSignatureHandler - SignaturesHandler process.SignaturesHandler - HeartbeatExpiryTimespanInSec int64 - MaxAllowedTrieNodeChunks uint32 - TrieNodeChunksInactivityTimeout time.Duration - MainPeerShardMapper process.PeerShardMapper - FullArchivePeerShardMapper process.PeerShardMapper - HardforkTrigger heartbeat.HardforkTrigger - NodeOperationMode common.NodeOperation - InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + CoreComponents process.CoreComponentsHolder + CryptoComponents process.CryptoComponentsHolder + Accounts state.AccountsAdapter + ShardCoordinator sharding.Coordinator + NodesCoordinator nodesCoordinator.NodesCoordinator + MainMessenger process.TopicHandler + FullArchiveMessenger process.TopicHandler + Store dataRetriever.StorageService + DataPool dataRetriever.PoolsHolder + MaxTxNonceDeltaAllowed int + TxFeeHandler process.FeeHandler + BlockBlackList process.TimeCacher + HeaderSigVerifier process.InterceptedHeaderSigVerifier + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + ValidityAttester process.ValidityAttester + EpochStartTrigger process.EpochStartTriggerHandler + WhiteListHandler process.WhiteListHandler + WhiteListerVerifiedTxs process.WhiteListHandler + AntifloodHandler process.P2PAntifloodHandler + ArgumentsParser process.ArgumentsParser + PreferredPeersHolder process.PreferredPeersHolderHandler + SizeCheckDelta uint32 + RequestHandler process.RequestHandler + PeerSignatureHandler crypto.PeerSignatureHandler + SignaturesHandler process.SignaturesHandler + HeartbeatExpiryTimespanInSec int64 + PeerAuthenticationTimeBetweenSendsInSec int64 + MaxAllowedTrieNodeChunks uint32 + TrieNodeChunksInactivityTimeout time.Duration + MainPeerShardMapper process.PeerShardMapper + FullArchivePeerShardMapper process.PeerShardMapper + HardforkTrigger heartbeat.HardforkTrigger + NodeOperationMode common.NodeOperation + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory } diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index 5c0dcda5405..90e6dc775e8 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -88,22 +88,24 @@ func NewMetaInterceptorsContainerFactory( } argInterceptorFactory := &interceptorFactory.ArgInterceptedDataFactory{ - CoreComponents: args.CoreComponents, - CryptoComponents: args.CryptoComponents, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: args.NodesCoordinator, - FeeHandler: args.TxFeeHandler, - WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - HeaderSigVerifier: args.HeaderSigVerifier, - ValidityAttester: args.ValidityAttester, - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - EpochStartTrigger: args.EpochStartTrigger, - ArgsParser: args.ArgumentsParser, - PeerSignatureHandler: args.PeerSignatureHandler, - SignaturesHandler: args.SignaturesHandler, - HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, - PeerID: args.MainMessenger.ID(), - PeerShardMapper: args.MainPeerShardMapper, + CoreComponents: args.CoreComponents, + CryptoComponents: args.CryptoComponents, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: args.NodesCoordinator, + FeeHandler: args.TxFeeHandler, + WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + HeaderSigVerifier: args.HeaderSigVerifier, + ValidityAttester: args.ValidityAttester, + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + EpochStartTrigger: args.EpochStartTrigger, + ArgsParser: args.ArgumentsParser, + PeerSignatureHandler: args.PeerSignatureHandler, + SignaturesHandler: args.SignaturesHandler, + HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, + PeerID: args.MainMessenger.ID(), + PeerShardMapper: args.MainPeerShardMapper, + PeerAuthCacher: args.DataPool.PeerAuthentications(), + PeerAuthenticationTimeBetweenSendsInSec: args.PeerAuthenticationTimeBetweenSendsInSec, } base := &baseInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go index 1d40dc9e81f..c009f734f20 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory_test.go @@ -707,37 +707,38 @@ func getArgumentsMeta( cryptoComp *mock.CryptoComponentsMock, ) interceptorscontainer.CommonInterceptorsContainerFactoryArgs { return interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComp, - CryptoComponents: cryptoComp, - Accounts: &stateMock.AccountsStub{}, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - MainMessenger: &mock.TopicHandlerStub{}, - FullArchiveMessenger: &mock.TopicHandlerStub{}, - Store: createMetaStore(), - DataPool: createMetaDataPools(), - MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: &testscommon.TimeCacheStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - ValidityAttester: &mock.ValidityAttesterStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - WhiteListHandler: &testscommon.WhiteListHandlerStub{}, - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - ArgumentsParser: &testscommon.ArgumentParserMock{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second, - MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, + CoreComponents: coreComp, + CryptoComponents: cryptoComp, + Accounts: &stateMock.AccountsStub{}, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + MainMessenger: &mock.TopicHandlerStub{}, + FullArchiveMessenger: &mock.TopicHandlerStub{}, + Store: createMetaStore(), + DataPool: createMetaDataPools(), + MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: &testscommon.TimeCacheStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + ValidityAttester: &mock.ValidityAttesterStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + ArgumentsParser: &testscommon.ArgumentParserMock{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerAuthenticationTimeBetweenSendsInSec: 60, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index c51a68f40ca..048d469485c 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -89,22 +89,24 @@ func NewShardInterceptorsContainerFactory( } argInterceptorFactory := &interceptorFactory.ArgInterceptedDataFactory{ - CoreComponents: args.CoreComponents, - CryptoComponents: args.CryptoComponents, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: args.NodesCoordinator, - FeeHandler: args.TxFeeHandler, - HeaderSigVerifier: args.HeaderSigVerifier, - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - ValidityAttester: args.ValidityAttester, - EpochStartTrigger: args.EpochStartTrigger, - WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - ArgsParser: args.ArgumentsParser, - PeerSignatureHandler: args.PeerSignatureHandler, - SignaturesHandler: args.SignaturesHandler, - HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, - PeerID: args.MainMessenger.ID(), - PeerShardMapper: args.MainPeerShardMapper, + CoreComponents: args.CoreComponents, + CryptoComponents: args.CryptoComponents, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: args.NodesCoordinator, + FeeHandler: args.TxFeeHandler, + HeaderSigVerifier: args.HeaderSigVerifier, + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + ValidityAttester: args.ValidityAttester, + EpochStartTrigger: args.EpochStartTrigger, + WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + ArgsParser: args.ArgumentsParser, + PeerSignatureHandler: args.PeerSignatureHandler, + SignaturesHandler: args.SignaturesHandler, + HeartbeatExpiryTimespanInSec: args.HeartbeatExpiryTimespanInSec, + PeerID: args.MainMessenger.ID(), + PeerShardMapper: args.MainPeerShardMapper, + PeerAuthCacher: args.DataPool.PeerAuthentications(), + PeerAuthenticationTimeBetweenSendsInSec: args.PeerAuthenticationTimeBetweenSendsInSec, } base := &baseInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go index 8d4520beff7..88881bf2f63 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory_test.go @@ -738,37 +738,38 @@ func getArgumentsShard( cryptoComp *mock.CryptoComponentsMock, ) interceptorscontainer.CommonInterceptorsContainerFactoryArgs { return interceptorscontainer.CommonInterceptorsContainerFactoryArgs{ - CoreComponents: coreComp, - CryptoComponents: cryptoComp, - Accounts: &stateMock.AccountsStub{}, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - MainMessenger: &mock.TopicHandlerStub{}, - FullArchiveMessenger: &mock.TopicHandlerStub{}, - Store: createShardStore(), - DataPool: createShardDataPools(), - MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, - TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, - BlockBlackList: &testscommon.TimeCacheStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - SizeCheckDelta: 0, - ValidityAttester: &mock.ValidityAttesterStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, - WhiteListHandler: &testscommon.WhiteListHandlerStub{}, - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - ArgumentsParser: &testscommon.ArgumentParserMock{}, - PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, - RequestHandler: &testscommon.RequestHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second, - MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, + CoreComponents: coreComp, + CryptoComponents: cryptoComp, + Accounts: &stateMock.AccountsStub{}, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + MainMessenger: &mock.TopicHandlerStub{}, + FullArchiveMessenger: &mock.TopicHandlerStub{}, + Store: createShardStore(), + DataPool: createShardDataPools(), + MaxTxNonceDeltaAllowed: maxTxNonceDeltaAllowed, + TxFeeHandler: &economicsmocks.EconomicsHandlerMock{}, + BlockBlackList: &testscommon.TimeCacheStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + SizeCheckDelta: 0, + ValidityAttester: &mock.ValidityAttesterStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + AntifloodHandler: &mock.P2PAntifloodHandlerStub{}, + WhiteListHandler: &testscommon.WhiteListHandlerStub{}, + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + ArgumentsParser: &testscommon.ArgumentParserMock{}, + PreferredPeersHolder: &p2pmocks.PeersHolderStub{}, + RequestHandler: &testscommon.RequestHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerAuthenticationTimeBetweenSendsInSec: 60, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + FullArchivePeerShardMapper: &p2pmocks.NetworkShardingCollectorStub{}, + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + InterceptedDataVerifierFactory: &mock.InterceptedDataVerifierFactoryMock{}, } } diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 9d49ac5ef5d..1606d8dea8b 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -10,35 +10,45 @@ import ( crypto "github.com/multiversx/mx-chain-crypto-go" "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" + "github.com/multiversx/mx-chain-go/storage" logger "github.com/multiversx/mx-chain-logger-go" ) +const ( + minPeerAuthenticationTimeBetweenSendsInSec = 1 + maxPercentAllowed = 0.8 // TODO: move this into config with supernova +) + // ArgInterceptedPeerAuthentication is the argument used in the intercepted peer authentication constructor type ArgInterceptedPeerAuthentication struct { ArgBaseInterceptedHeartbeat - NodesCoordinator NodesCoordinator - SignaturesHandler SignaturesHandler - PeerSignatureHandler crypto.PeerSignatureHandler - PayloadValidator process.PeerAuthenticationPayloadValidator - HardforkTriggerPubKey []byte - PeerShardMapper process.PeerShardMapper - MessageOriginator core.PeerID - SelfPeerID core.PeerID + NodesCoordinator NodesCoordinator + SignaturesHandler SignaturesHandler + PeerSignatureHandler crypto.PeerSignatureHandler + PayloadValidator process.PeerAuthenticationPayloadValidator + HardforkTriggerPubKey []byte + PeerShardMapper process.PeerShardMapper + PeerAuthCacher storage.Cacher + MessageOriginator core.PeerID + SelfPeerID core.PeerID + PeerAuthenticationTimeBetweenSendsInSec int64 } // interceptedPeerAuthentication is a wrapper over PeerAuthentication type interceptedPeerAuthentication struct { - peerAuthentication heartbeat.PeerAuthentication - payload heartbeat.Payload - peerId core.PeerID - nodesCoordinator NodesCoordinator - signaturesHandler SignaturesHandler - peerSignatureHandler crypto.PeerSignatureHandler - payloadValidator process.PeerAuthenticationPayloadValidator - hardforkTriggerPubKey []byte - peerShardMapper process.PeerShardMapper - messageOriginator core.PeerID - selfPeerID core.PeerID + peerAuthentication heartbeat.PeerAuthentication + payload heartbeat.Payload + peerId core.PeerID + nodesCoordinator NodesCoordinator + signaturesHandler SignaturesHandler + peerSignatureHandler crypto.PeerSignatureHandler + payloadValidator process.PeerAuthenticationPayloadValidator + hardforkTriggerPubKey []byte + peerShardMapper process.PeerShardMapper + peerAuthCacher storage.Cacher + messageOriginator core.PeerID + selfPeerID core.PeerID + peerAuthenticationTimeBetweenSendsInSec int64 } // NewInterceptedPeerAuthentication tries to create a new intercepted peer authentication instance @@ -54,16 +64,18 @@ func NewInterceptedPeerAuthentication(arg ArgInterceptedPeerAuthentication) (*in } intercepted := &interceptedPeerAuthentication{ - peerAuthentication: *peerAuthentication, - payload: *payload, - nodesCoordinator: arg.NodesCoordinator, - signaturesHandler: arg.SignaturesHandler, - peerSignatureHandler: arg.PeerSignatureHandler, - payloadValidator: arg.PayloadValidator, - hardforkTriggerPubKey: arg.HardforkTriggerPubKey, - peerShardMapper: arg.PeerShardMapper, - messageOriginator: arg.MessageOriginator, - selfPeerID: arg.SelfPeerID, + peerAuthentication: *peerAuthentication, + payload: *payload, + nodesCoordinator: arg.NodesCoordinator, + signaturesHandler: arg.SignaturesHandler, + peerSignatureHandler: arg.PeerSignatureHandler, + payloadValidator: arg.PayloadValidator, + hardforkTriggerPubKey: arg.HardforkTriggerPubKey, + peerShardMapper: arg.PeerShardMapper, + peerAuthCacher: arg.PeerAuthCacher, + messageOriginator: arg.MessageOriginator, + selfPeerID: arg.SelfPeerID, + peerAuthenticationTimeBetweenSendsInSec: arg.PeerAuthenticationTimeBetweenSendsInSec, } intercepted.peerId = core.PeerID(intercepted.peerAuthentication.Pid) @@ -93,6 +105,12 @@ func checkArg(arg ArgInterceptedPeerAuthentication) error { if check.IfNil(arg.PeerShardMapper) { return process.ErrNilPeerShardMapper } + if check.IfNil(arg.PeerAuthCacher) { + return process.ErrNilPeerAuthenticationCacher + } + if arg.PeerAuthenticationTimeBetweenSendsInSec < minPeerAuthenticationTimeBetweenSendsInSec { + return fmt.Errorf("%w for PeerAuthenticationTimeBetweenSendsInSec", process.ErrInvalidValue) + } return nil } @@ -148,16 +166,19 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { // Early exit if mapping already exists and the message is from itself existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) + hasInCache := ipa.peerAuthCacher.Has(ipa.Pubkey()) isFromSelf := ipa.messageOriginator == ipa.selfPeerID pairExists := string(existingInfo.PkBytes) == string(ipa.Pubkey()) if pairExists && isFromSelf { return nil } - if pairExists && !isFromSelf { + if pairExists && !isFromSelf && hasInCache { return process.ErrPeerAlreadyAuthenticated } - if existingInfo.AuthTimestamp > ipa.payload.Timestamp { + deltaSec := float64(ipa.peerAuthenticationTimeBetweenSendsInSec) * maxPercentAllowed + minTimestampAccepted := existingInfo.AuthTimestamp + int64(deltaSec) + if ipa.payload.Timestamp <= minTimestampAccepted { return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthTimestamp) } } diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index c6ce19ca8c2..606e44cc973 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -15,6 +15,7 @@ import ( processMocks "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" "github.com/stretchr/testify/assert" @@ -55,12 +56,14 @@ func createMockInterceptedPeerAuthenticationArg(interceptedData *heartbeat.PeerA ArgBaseInterceptedHeartbeat: ArgBaseInterceptedHeartbeat{ Marshaller: &marshal.GogoProtoMarshalizer{}, }, - NodesCoordinator: &shardingMocks.NodesCoordinatorStub{}, - SignaturesHandler: &processMocks.SignaturesHandlerStub{}, - PeerSignatureHandler: &cryptoMocks.PeerSignatureHandlerStub{}, - PayloadValidator: &testscommon.PeerAuthenticationPayloadValidatorStub{}, - HardforkTriggerPubKey: providedHardforkPubKey, - PeerShardMapper: &processMocks.PeerShardMapperStub{}, + NodesCoordinator: &shardingMocks.NodesCoordinatorStub{}, + SignaturesHandler: &processMocks.SignaturesHandlerStub{}, + PeerSignatureHandler: &cryptoMocks.PeerSignatureHandlerStub{}, + PayloadValidator: &testscommon.PeerAuthenticationPayloadValidatorStub{}, + HardforkTriggerPubKey: providedHardforkPubKey, + PeerShardMapper: &processMocks.PeerShardMapperStub{}, + PeerAuthCacher: cache.NewCacherStub(), + PeerAuthenticationTimeBetweenSendsInSec: 10, } arg.DataBuff, _ = arg.Marshaller.Marshal(interceptedData) @@ -140,6 +143,26 @@ func TestNewInterceptedPeerAuthentication(t *testing.T) { assert.True(t, check.IfNil(ipa)) assert.Equal(t, process.ErrNilPeerShardMapper, err) }) + t.Run("nil peer auth cacher should error", func(t *testing.T) { + t.Parallel() + + arg := createMockInterceptedPeerAuthenticationArg(createDefaultInterceptedPeerAuthentication()) + arg.PeerAuthCacher = nil + + ipa, err := NewInterceptedPeerAuthentication(arg) + assert.True(t, check.IfNil(ipa)) + assert.Equal(t, process.ErrNilPeerAuthenticationCacher, err) + }) + t.Run("invalid peer auth time between sends should error", func(t *testing.T) { + t.Parallel() + + arg := createMockInterceptedPeerAuthenticationArg(createDefaultInterceptedPeerAuthentication()) + arg.PeerAuthenticationTimeBetweenSendsInSec = 0 + + ipa, err := NewInterceptedPeerAuthentication(arg) + assert.True(t, check.IfNil(ipa)) + assert.ErrorIs(t, err, process.ErrInvalidValue) + }) t.Run("unmarshal returns error", func(t *testing.T) { t.Parallel() @@ -268,7 +291,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err = ipa.CheckValidity() assert.True(t, errors.Is(err, expectedErr)) }) - t.Run("peer already authenticated with same pubkey should early exit, message from self", func(t *testing.T) { + t.Run("peer already authenticated with same pubkey should return error", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() @@ -292,33 +315,7 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.NoError(t, err) }) - t.Run("peer already authenticated with same pubkey should return error, message not from self", func(t *testing.T) { - t.Parallel() - - providedPA := createDefaultInterceptedPeerAuthentication() - arg := createMockInterceptedPeerAuthenticationArg(providedPA) - arg.MessageOriginator = "originator" - arg.SelfPeerID = "self" - - arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ - VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { - require.Fail(t, "should have not been called") - return expectedErr - }, - } - arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ - GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { - return core.P2PPeerInfo{ - PkBytes: providedPA.Pubkey, - } - }, - } - - ipa, _ := NewInterceptedPeerAuthentication(arg) - err := ipa.CheckValidity() - assert.Equal(t, process.ErrPeerAlreadyAuthenticated, err) - }) - t.Run("peer already authenticated with newer timestamp should return error", func(t *testing.T) { + t.Run("peer already authenticated with newer timestamp should early exit", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() diff --git a/process/heartbeat/validator/peerAuthenticationPayloadValidator.go b/process/heartbeat/validator/peerAuthenticationPayloadValidator.go index cb220a2e482..c2cb7d7f90a 100644 --- a/process/heartbeat/validator/peerAuthenticationPayloadValidator.go +++ b/process/heartbeat/validator/peerAuthenticationPayloadValidator.go @@ -8,28 +8,36 @@ import ( ) const ( + minDurationInSec = 10 payloadExpiryThresholdInSec = 10 ) type peerAuthenticationPayloadValidator struct { - getTimeHandler func() time.Time + expiryTimespanInSec int64 + getTimeHandler func() time.Time } // NewPeerAuthenticationPayloadValidator creates a new peer authentication payload validator instance -func NewPeerAuthenticationPayloadValidator() (*peerAuthenticationPayloadValidator, error) { +func NewPeerAuthenticationPayloadValidator(expiryTimespanInSec int64) (*peerAuthenticationPayloadValidator, error) { + if expiryTimespanInSec < minDurationInSec { + return nil, process.ErrInvalidExpiryTimespan + } + return &peerAuthenticationPayloadValidator{ - getTimeHandler: time.Now, + expiryTimespanInSec: expiryTimespanInSec, + getTimeHandler: time.Now, }, nil } // ValidateTimestamp will return an error if the provided payload timestamp is not valid func (validator *peerAuthenticationPayloadValidator) ValidateTimestamp(payloadTimestamp int64) error { currentTimeStamp := validator.getTimeHandler().Unix() + minTimestampAllowed := currentTimeStamp - validator.expiryTimespanInSec maxTimestampAllowed := currentTimeStamp + payloadExpiryThresholdInSec - if payloadTimestamp > maxTimestampAllowed { - return fmt.Errorf("%w message time stamp: %v, maximum: %v", - process.ErrMessageExpired, payloadTimestamp, maxTimestampAllowed) + if payloadTimestamp < minTimestampAllowed || payloadTimestamp > maxTimestampAllowed { + return fmt.Errorf("%w message time stamp: %v, minimum: %v, maximum: %v", + process.ErrMessageExpired, payloadTimestamp, minTimestampAllowed, maxTimestampAllowed) } return nil diff --git a/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go b/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go index 3592127a278..1d924908012 100644 --- a/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go +++ b/process/heartbeat/validator/peerAuthenticationPayloadValidator_test.go @@ -10,6 +10,33 @@ import ( "github.com/stretchr/testify/assert" ) +func TestNewPeerAuthenticationPayloadValidator(t *testing.T) { + t.Parallel() + + t.Run("invalid expiry duration should error", func(t *testing.T) { + t.Parallel() + + valsToTest := int64(100) + + for i := int64(1); i <= valsToTest; i++ { + validator, err := NewPeerAuthenticationPayloadValidator(minDurationInSec - i) + assert.True(t, check.IfNil(validator)) + assert.Equal(t, process.ErrInvalidExpiryTimespan, err) + } + }) + t.Run("should work", func(t *testing.T) { + t.Parallel() + + valsToTest := int64(100) + + for i := int64(0); i < valsToTest; i++ { + validator, err := NewPeerAuthenticationPayloadValidator(minDurationInSec + i) + assert.False(t, check.IfNil(validator)) + assert.Nil(t, err) + } + }) +} + func TestPeerAuthenticationPayloadValidator_ValidateTimestamp(t *testing.T) { t.Parallel() @@ -17,15 +44,36 @@ func TestPeerAuthenticationPayloadValidator_ValidateTimestamp(t *testing.T) { t.Parallel() currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator() - assert.False(t, check.IfNil(validator)) + validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) assert.Nil(t, validator.ValidateTimestamp(currentTime.Unix())) }) + t.Run("payload time stamp is exactly the minim accepted", func(t *testing.T) { + t.Parallel() + + currentTime := time.Now() + validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) + validator.getTimeHandler = func() time.Time { + return currentTime.Add(time.Second * 1120) + } + minimumAccepted := currentTime.Add(time.Second * (1120 - minDurationInSec)) + assert.Nil(t, validator.ValidateTimestamp(minimumAccepted.Unix())) + }) + t.Run("payload time stamp is less than minim accepted", func(t *testing.T) { + t.Parallel() + + currentTime := time.Now() + validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) + validator.getTimeHandler = func() time.Time { + return currentTime.Add(time.Second * 1120) + } + minimumAccepted := currentTime.Add(time.Second * (1120 - minDurationInSec - 1)) + assert.True(t, errors.Is(validator.ValidateTimestamp(minimumAccepted.Unix()), process.ErrMessageExpired)) + }) t.Run("payload time stamp is exactly the maximum accepted", func(t *testing.T) { t.Parallel() currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator() + validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) validator.getTimeHandler = func() time.Time { return currentTime.Add(time.Second * 1120) } @@ -36,7 +84,7 @@ func TestPeerAuthenticationPayloadValidator_ValidateTimestamp(t *testing.T) { t.Parallel() currentTime := time.Now() - validator, _ := NewPeerAuthenticationPayloadValidator() + validator, _ := NewPeerAuthenticationPayloadValidator(minDurationInSec) validator.getTimeHandler = func() time.Time { return currentTime.Add(time.Second * 1120) } diff --git a/process/interceptors/factory/argInterceptedDataFactory.go b/process/interceptors/factory/argInterceptedDataFactory.go index cb6d263e2b7..4f50225091f 100644 --- a/process/interceptors/factory/argInterceptedDataFactory.go +++ b/process/interceptors/factory/argInterceptedDataFactory.go @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" crypto "github.com/multiversx/mx-chain-crypto-go" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" @@ -45,20 +46,22 @@ type interceptedDataCryptoComponentsHolder interface { // ArgInterceptedDataFactory holds all dependencies required by the shard and meta intercepted data factory in order to create // new instances type ArgInterceptedDataFactory struct { - CoreComponents interceptedDataCoreComponentsHolder - CryptoComponents interceptedDataCryptoComponentsHolder - ShardCoordinator sharding.Coordinator - NodesCoordinator nodesCoordinator.NodesCoordinator - FeeHandler process.FeeHandler - WhiteListerVerifiedTxs process.WhiteListHandler - HeaderSigVerifier process.InterceptedHeaderSigVerifier - ValidityAttester process.ValidityAttester - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - EpochStartTrigger process.EpochStartTriggerHandler - ArgsParser process.ArgumentsParser - PeerSignatureHandler crypto.PeerSignatureHandler - SignaturesHandler process.SignaturesHandler - HeartbeatExpiryTimespanInSec int64 - PeerID core.PeerID - PeerShardMapper process.PeerShardMapper + CoreComponents interceptedDataCoreComponentsHolder + CryptoComponents interceptedDataCryptoComponentsHolder + ShardCoordinator sharding.Coordinator + NodesCoordinator nodesCoordinator.NodesCoordinator + FeeHandler process.FeeHandler + WhiteListerVerifiedTxs process.WhiteListHandler + HeaderSigVerifier process.InterceptedHeaderSigVerifier + ValidityAttester process.ValidityAttester + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + EpochStartTrigger process.EpochStartTriggerHandler + ArgsParser process.ArgumentsParser + PeerSignatureHandler crypto.PeerSignatureHandler + SignaturesHandler process.SignaturesHandler + HeartbeatExpiryTimespanInSec int64 + PeerID core.PeerID + PeerShardMapper process.PeerShardMapper + PeerAuthCacher storage.Cacher + PeerAuthenticationTimeBetweenSendsInSec int64 } diff --git a/process/interceptors/factory/interceptedEquivalentProofsFactory_test.go b/process/interceptors/factory/interceptedEquivalentProofsFactory_test.go index 74e0aa278bd..1edb7e73f65 100644 --- a/process/interceptors/factory/interceptedEquivalentProofsFactory_test.go +++ b/process/interceptors/factory/interceptedEquivalentProofsFactory_test.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/pool" "github.com/stretchr/testify/require" @@ -25,9 +26,11 @@ func createMockArgInterceptedEquivalentProofsFactory() ArgInterceptedEquivalentP Hash: &hashingMocks.HasherMock{}, FieldsSizeCheckerField: &testscommon.FieldsSizeCheckerMock{}, }, - ShardCoordinator: &mock.ShardCoordinatorMock{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - NodesCoordinator: &shardingMocks.NodesCoordinatorStub{}, + ShardCoordinator: &mock.ShardCoordinatorMock{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + NodesCoordinator: &shardingMocks.NodesCoordinatorStub{}, + PeerAuthCacher: cache.NewCacherStub(), + PeerAuthenticationTimeBetweenSendsInSec: 10, }, ProofsPool: &dataRetriever.ProofsPoolMock{}, HeadersPool: &pool.HeadersPoolStub{}, diff --git a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go index 6890990296b..45e7493f2e3 100644 --- a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go +++ b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/versioning" "github.com/multiversx/mx-chain-core-go/data/block" crypto "github.com/multiversx/mx-chain-crypto-go" + "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/stretchr/testify/assert" "github.com/multiversx/mx-chain-go/common/graceperiod" @@ -99,21 +100,23 @@ func createMockArgMetaHeaderFactoryArgument( ) *ArgInterceptedMetaHeaderFactory { return &ArgInterceptedMetaHeaderFactory{ ArgInterceptedDataFactory: ArgInterceptedDataFactory{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - FeeHandler: createMockFeeHandler(), - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - ValidityAttester: &mock.ValidityAttesterStub{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - ArgsParser: &testscommon.ArgumentParserMock{}, - PeerSignatureHandler: &processMocks.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMocks.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - PeerID: "pid", + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + FeeHandler: createMockFeeHandler(), + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + ValidityAttester: &mock.ValidityAttesterStub{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + ArgsParser: &testscommon.ArgumentParserMock{}, + PeerSignatureHandler: &processMocks.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMocks.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerID: "pid", + PeerAuthCacher: cache.NewCacherStub(), + PeerAuthenticationTimeBetweenSendsInSec: 60, }, } } @@ -123,22 +126,24 @@ func createMockArgument( cryptoComponents *mock.CryptoComponentsMock, ) *ArgInterceptedDataFactory { return &ArgInterceptedDataFactory{ - CoreComponents: coreComponents, - CryptoComponents: cryptoComponents, - ShardCoordinator: mock.NewOneShardCoordinatorMock(), - NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), - FeeHandler: createMockFeeHandler(), - WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, - HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, - ValidityAttester: &mock.ValidityAttesterStub{}, - HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, - EpochStartTrigger: &mock.EpochStartTriggerStub{}, - ArgsParser: &testscommon.ArgumentParserMock{}, - PeerSignatureHandler: &processMocks.PeerSignatureHandlerStub{}, - SignaturesHandler: &processMocks.SignaturesHandlerStub{}, - HeartbeatExpiryTimespanInSec: 30, - PeerID: "pid", - PeerShardMapper: &processMocks.PeerShardMapperStub{}, + CoreComponents: coreComponents, + CryptoComponents: cryptoComponents, + ShardCoordinator: mock.NewOneShardCoordinatorMock(), + NodesCoordinator: shardingMocks.NewNodesCoordinatorMock(), + FeeHandler: createMockFeeHandler(), + WhiteListerVerifiedTxs: &testscommon.WhiteListHandlerStub{}, + HeaderSigVerifier: &consensus.HeaderSigVerifierMock{}, + ValidityAttester: &mock.ValidityAttesterStub{}, + HeaderIntegrityVerifier: &mock.HeaderIntegrityVerifierStub{}, + EpochStartTrigger: &mock.EpochStartTriggerStub{}, + ArgsParser: &testscommon.ArgumentParserMock{}, + PeerSignatureHandler: &processMocks.PeerSignatureHandlerStub{}, + SignaturesHandler: &processMocks.SignaturesHandlerStub{}, + HeartbeatExpiryTimespanInSec: 30, + PeerID: "pid", + PeerShardMapper: &processMocks.PeerShardMapperStub{}, + PeerAuthCacher: cache.NewCacherStub(), + PeerAuthenticationTimeBetweenSendsInSec: 60, } } diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index b3dfb78c434..6985d4a8a96 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -10,18 +10,21 @@ import ( "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/heartbeat" "github.com/multiversx/mx-chain-go/process/heartbeat/validator" + "github.com/multiversx/mx-chain-go/storage" ) const minDurationInSec = 10 type interceptedPeerAuthenticationDataFactory struct { - marshalizer marshal.Marshalizer - nodesCoordinator heartbeat.NodesCoordinator - signaturesHandler heartbeat.SignaturesHandler - peerSignatureHandler crypto.PeerSignatureHandler - hardforkTriggerPubKey []byte - payloadValidator process.PeerAuthenticationPayloadValidator - peerShardMapper process.PeerShardMapper + marshalizer marshal.Marshalizer + nodesCoordinator heartbeat.NodesCoordinator + signaturesHandler heartbeat.SignaturesHandler + peerSignatureHandler crypto.PeerSignatureHandler + hardforkTriggerPubKey []byte + payloadValidator process.PeerAuthenticationPayloadValidator + peerShardMapper process.PeerShardMapper + peerAuthCacher storage.Cacher + peerAuthenticationTimeBetweenSendsInSec int64 } // NewInterceptedPeerAuthenticationDataFactory creates an instance of interceptedPeerAuthenticationDataFactory @@ -31,19 +34,21 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) return nil, err } - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator() + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(arg.HeartbeatExpiryTimespanInSec) if err != nil { return nil, err } return &interceptedPeerAuthenticationDataFactory{ - marshalizer: arg.CoreComponents.InternalMarshalizer(), - nodesCoordinator: arg.NodesCoordinator, - signaturesHandler: arg.SignaturesHandler, - peerSignatureHandler: arg.PeerSignatureHandler, - payloadValidator: payloadValidator, - hardforkTriggerPubKey: arg.CoreComponents.HardforkTriggerPubKey(), - peerShardMapper: arg.PeerShardMapper, + marshalizer: arg.CoreComponents.InternalMarshalizer(), + nodesCoordinator: arg.NodesCoordinator, + signaturesHandler: arg.SignaturesHandler, + peerSignatureHandler: arg.PeerSignatureHandler, + payloadValidator: payloadValidator, + hardforkTriggerPubKey: arg.CoreComponents.HardforkTriggerPubKey(), + peerShardMapper: arg.PeerShardMapper, + peerAuthCacher: arg.PeerAuthCacher, + peerAuthenticationTimeBetweenSendsInSec: arg.PeerAuthenticationTimeBetweenSendsInSec, }, nil } @@ -74,19 +79,20 @@ func checkArgInterceptedDataFactory(args ArgInterceptedDataFactory) error { } // Create creates instances of InterceptedData by unmarshalling provided buffer -func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, messageOriginator core.PeerID) (process.InterceptedData, error) { +func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ core.PeerID) (process.InterceptedData, error) { arg := heartbeat.ArgInterceptedPeerAuthentication{ ArgBaseInterceptedHeartbeat: heartbeat.ArgBaseInterceptedHeartbeat{ DataBuff: buff, Marshaller: ipadf.marshalizer, }, - NodesCoordinator: ipadf.nodesCoordinator, - SignaturesHandler: ipadf.signaturesHandler, - PeerSignatureHandler: ipadf.peerSignatureHandler, - PayloadValidator: ipadf.payloadValidator, - HardforkTriggerPubKey: ipadf.hardforkTriggerPubKey, - PeerShardMapper: ipadf.peerShardMapper, - MessageOriginator: messageOriginator, + NodesCoordinator: ipadf.nodesCoordinator, + SignaturesHandler: ipadf.signaturesHandler, + PeerSignatureHandler: ipadf.peerSignatureHandler, + PayloadValidator: ipadf.payloadValidator, + HardforkTriggerPubKey: ipadf.hardforkTriggerPubKey, + PeerShardMapper: ipadf.peerShardMapper, + PeerAuthCacher: ipadf.peerAuthCacher, + PeerAuthenticationTimeBetweenSendsInSec: ipadf.peerAuthenticationTimeBetweenSendsInSec, } return heartbeat.NewInterceptedPeerAuthentication(arg) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 865d3570463..aa1abb4ed50 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -51,18 +51,20 @@ func createInterceptedPeerAuthentication() *heartbeatMessages.PeerAuthentication } func createMockInterceptedPeerAuthentication() process.InterceptedData { - payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator() + payloadValidator, _ := validator.NewPeerAuthenticationPayloadValidator(30) arg := heartbeat.ArgInterceptedPeerAuthentication{ ArgBaseInterceptedHeartbeat: heartbeat.ArgBaseInterceptedHeartbeat{ Marshaller: &mock.MarshalizerMock{}, }, - NodesCoordinator: &mock.NodesCoordinatorStub{}, - SignaturesHandler: &mock.SignaturesHandlerStub{}, - PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, - PayloadValidator: payloadValidator, - HardforkTriggerPubKey: []byte("provided hardfork pub key"), - PeerShardMapper: &mock.PeerShardMapperStub{}, + NodesCoordinator: &mock.NodesCoordinatorStub{}, + SignaturesHandler: &mock.SignaturesHandlerStub{}, + PeerSignatureHandler: &mock.PeerSignatureHandlerStub{}, + PayloadValidator: payloadValidator, + HardforkTriggerPubKey: []byte("provided hardfork pub key"), + PeerShardMapper: &mock.PeerShardMapperStub{}, + PeerAuthCacher: cache.NewCacherStub(), + PeerAuthenticationTimeBetweenSendsInSec: 10, } arg.DataBuff, _ = arg.Marshaller.Marshal(createInterceptedPeerAuthentication()) ipa, _ := heartbeat.NewInterceptedPeerAuthentication(arg) diff --git a/update/factory/exportHandlerFactory.go b/update/factory/exportHandlerFactory.go index 0cda7a5d2e0..503534aa009 100644 --- a/update/factory/exportHandlerFactory.go +++ b/update/factory/exportHandlerFactory.go @@ -38,79 +38,81 @@ var log = logger.GetOrCreate("update/factory") // ArgsExporter is the argument structure to create a new exporter type ArgsExporter struct { - CoreComponents process.CoreComponentsHolder - CryptoComponents process.CryptoComponentsHolder - StatusCoreComponents process.StatusCoreComponentsHolder - NetworkComponents mxFactory.NetworkComponentsHolder - HeaderValidator epochStart.HeaderValidator - DataPool dataRetriever.PoolsHolder - StorageService dataRetriever.StorageService - RequestHandler process.RequestHandler - ShardCoordinator sharding.Coordinator - ActiveAccountsDBs map[state.AccountsDbIdentifier]state.AccountsAdapter - ExistingResolvers dataRetriever.ResolversContainer - ExistingRequesters dataRetriever.RequestersContainer - ExportFolder string - ExportTriesStorageConfig config.StorageConfig - ExportStateStorageConfig config.StorageConfig - ExportStateKeysConfig config.StorageConfig - MaxTrieLevelInMemory uint - WhiteListHandler process.WhiteListHandler - WhiteListerVerifiedTxs process.WhiteListHandler - MainInterceptorsContainer process.InterceptorsContainer - FullArchiveInterceptorsContainer process.InterceptorsContainer - NodesCoordinator nodesCoordinator.NodesCoordinator - HeaderSigVerifier process.InterceptedHeaderSigVerifier - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - ValidityAttester process.ValidityAttester - RoundHandler process.RoundHandler - InterceptorDebugConfig config.InterceptorResolverDebugConfig - MaxHardCapForMissingNodes int - NumConcurrentTrieSyncers int - TrieSyncerVersion int - CheckNodesOnDisk bool - NodeOperationMode common.NodeOperation - InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + CoreComponents process.CoreComponentsHolder + CryptoComponents process.CryptoComponentsHolder + StatusCoreComponents process.StatusCoreComponentsHolder + NetworkComponents mxFactory.NetworkComponentsHolder + HeaderValidator epochStart.HeaderValidator + DataPool dataRetriever.PoolsHolder + StorageService dataRetriever.StorageService + RequestHandler process.RequestHandler + ShardCoordinator sharding.Coordinator + ActiveAccountsDBs map[state.AccountsDbIdentifier]state.AccountsAdapter + ExistingResolvers dataRetriever.ResolversContainer + ExistingRequesters dataRetriever.RequestersContainer + ExportFolder string + ExportTriesStorageConfig config.StorageConfig + ExportStateStorageConfig config.StorageConfig + ExportStateKeysConfig config.StorageConfig + MaxTrieLevelInMemory uint + WhiteListHandler process.WhiteListHandler + WhiteListerVerifiedTxs process.WhiteListHandler + MainInterceptorsContainer process.InterceptorsContainer + FullArchiveInterceptorsContainer process.InterceptorsContainer + NodesCoordinator nodesCoordinator.NodesCoordinator + HeaderSigVerifier process.InterceptedHeaderSigVerifier + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + ValidityAttester process.ValidityAttester + RoundHandler process.RoundHandler + InterceptorDebugConfig config.InterceptorResolverDebugConfig + MaxHardCapForMissingNodes int + NumConcurrentTrieSyncers int + TrieSyncerVersion int + CheckNodesOnDisk bool + NodeOperationMode common.NodeOperation + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + PeerAuthenticationTimeBetweenSendsInSec int64 } type exportHandlerFactory struct { - coreComponents process.CoreComponentsHolder - cryptoComponents process.CryptoComponentsHolder - statusCoreComponents process.StatusCoreComponentsHolder - networkComponents mxFactory.NetworkComponentsHolder - headerValidator epochStart.HeaderValidator - dataPool dataRetriever.PoolsHolder - storageService dataRetriever.StorageService - requestHandler process.RequestHandler - shardCoordinator sharding.Coordinator - activeAccountsDBs map[state.AccountsDbIdentifier]state.AccountsAdapter - exportFolder string - exportTriesStorageConfig config.StorageConfig - exportStateStorageConfig config.StorageConfig - exportStateKeysConfig config.StorageConfig - maxTrieLevelInMemory uint - whiteListHandler process.WhiteListHandler - whiteListerVerifiedTxs process.WhiteListHandler - mainInterceptorsContainer process.InterceptorsContainer - fullArchiveInterceptorsContainer process.InterceptorsContainer - existingResolvers dataRetriever.ResolversContainer - existingRequesters dataRetriever.RequestersContainer - epochStartTrigger epochStart.TriggerHandler - accounts state.AccountsAdapter - nodesCoordinator nodesCoordinator.NodesCoordinator - headerSigVerifier process.InterceptedHeaderSigVerifier - headerIntegrityVerifier process.HeaderIntegrityVerifier - validityAttester process.ValidityAttester - resolverContainer dataRetriever.ResolversContainer - requestersContainer dataRetriever.RequestersContainer - roundHandler process.RoundHandler - interceptorDebugConfig config.InterceptorResolverDebugConfig - maxHardCapForMissingNodes int - numConcurrentTrieSyncers int - trieSyncerVersion int - checkNodesOnDisk bool - nodeOperationMode common.NodeOperation - interceptedDataVerifierFactory process.InterceptedDataVerifierFactory + coreComponents process.CoreComponentsHolder + cryptoComponents process.CryptoComponentsHolder + statusCoreComponents process.StatusCoreComponentsHolder + networkComponents mxFactory.NetworkComponentsHolder + headerValidator epochStart.HeaderValidator + dataPool dataRetriever.PoolsHolder + storageService dataRetriever.StorageService + requestHandler process.RequestHandler + shardCoordinator sharding.Coordinator + activeAccountsDBs map[state.AccountsDbIdentifier]state.AccountsAdapter + exportFolder string + exportTriesStorageConfig config.StorageConfig + exportStateStorageConfig config.StorageConfig + exportStateKeysConfig config.StorageConfig + maxTrieLevelInMemory uint + whiteListHandler process.WhiteListHandler + whiteListerVerifiedTxs process.WhiteListHandler + mainInterceptorsContainer process.InterceptorsContainer + fullArchiveInterceptorsContainer process.InterceptorsContainer + existingResolvers dataRetriever.ResolversContainer + existingRequesters dataRetriever.RequestersContainer + epochStartTrigger epochStart.TriggerHandler + accounts state.AccountsAdapter + nodesCoordinator nodesCoordinator.NodesCoordinator + headerSigVerifier process.InterceptedHeaderSigVerifier + headerIntegrityVerifier process.HeaderIntegrityVerifier + validityAttester process.ValidityAttester + resolverContainer dataRetriever.ResolversContainer + requestersContainer dataRetriever.RequestersContainer + roundHandler process.RoundHandler + interceptorDebugConfig config.InterceptorResolverDebugConfig + maxHardCapForMissingNodes int + numConcurrentTrieSyncers int + trieSyncerVersion int + checkNodesOnDisk bool + nodeOperationMode common.NodeOperation + interceptedDataVerifierFactory process.InterceptedDataVerifierFactory + peerAuthenticationTimeBetweenSendsInSec int64 } // NewExportHandlerFactory creates an exporter factory @@ -236,40 +238,41 @@ func NewExportHandlerFactory(args ArgsExporter) (*exportHandlerFactory, error) { } e := &exportHandlerFactory{ - coreComponents: args.CoreComponents, - cryptoComponents: args.CryptoComponents, - networkComponents: args.NetworkComponents, - headerValidator: args.HeaderValidator, - dataPool: args.DataPool, - storageService: args.StorageService, - requestHandler: args.RequestHandler, - shardCoordinator: args.ShardCoordinator, - activeAccountsDBs: args.ActiveAccountsDBs, - exportFolder: args.ExportFolder, - exportTriesStorageConfig: args.ExportTriesStorageConfig, - exportStateStorageConfig: args.ExportStateStorageConfig, - exportStateKeysConfig: args.ExportStateKeysConfig, - mainInterceptorsContainer: args.MainInterceptorsContainer, - fullArchiveInterceptorsContainer: args.FullArchiveInterceptorsContainer, - whiteListHandler: args.WhiteListHandler, - whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - existingResolvers: args.ExistingResolvers, - existingRequesters: args.ExistingRequesters, - accounts: args.ActiveAccountsDBs[state.UserAccountsState], - nodesCoordinator: args.NodesCoordinator, - headerSigVerifier: args.HeaderSigVerifier, - headerIntegrityVerifier: args.HeaderIntegrityVerifier, - validityAttester: args.ValidityAttester, - maxTrieLevelInMemory: args.MaxTrieLevelInMemory, - roundHandler: args.RoundHandler, - interceptorDebugConfig: args.InterceptorDebugConfig, - maxHardCapForMissingNodes: args.MaxHardCapForMissingNodes, - numConcurrentTrieSyncers: args.NumConcurrentTrieSyncers, - trieSyncerVersion: args.TrieSyncerVersion, - checkNodesOnDisk: args.CheckNodesOnDisk, - statusCoreComponents: args.StatusCoreComponents, - nodeOperationMode: args.NodeOperationMode, - interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + coreComponents: args.CoreComponents, + cryptoComponents: args.CryptoComponents, + networkComponents: args.NetworkComponents, + headerValidator: args.HeaderValidator, + dataPool: args.DataPool, + storageService: args.StorageService, + requestHandler: args.RequestHandler, + shardCoordinator: args.ShardCoordinator, + activeAccountsDBs: args.ActiveAccountsDBs, + exportFolder: args.ExportFolder, + exportTriesStorageConfig: args.ExportTriesStorageConfig, + exportStateStorageConfig: args.ExportStateStorageConfig, + exportStateKeysConfig: args.ExportStateKeysConfig, + mainInterceptorsContainer: args.MainInterceptorsContainer, + fullArchiveInterceptorsContainer: args.FullArchiveInterceptorsContainer, + whiteListHandler: args.WhiteListHandler, + whiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + existingResolvers: args.ExistingResolvers, + existingRequesters: args.ExistingRequesters, + accounts: args.ActiveAccountsDBs[state.UserAccountsState], + nodesCoordinator: args.NodesCoordinator, + headerSigVerifier: args.HeaderSigVerifier, + headerIntegrityVerifier: args.HeaderIntegrityVerifier, + validityAttester: args.ValidityAttester, + maxTrieLevelInMemory: args.MaxTrieLevelInMemory, + roundHandler: args.RoundHandler, + interceptorDebugConfig: args.InterceptorDebugConfig, + maxHardCapForMissingNodes: args.MaxHardCapForMissingNodes, + numConcurrentTrieSyncers: args.NumConcurrentTrieSyncers, + trieSyncerVersion: args.TrieSyncerVersion, + checkNodesOnDisk: args.CheckNodesOnDisk, + statusCoreComponents: args.StatusCoreComponents, + nodeOperationMode: args.NodeOperationMode, + interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, + peerAuthenticationTimeBetweenSendsInSec: args.PeerAuthenticationTimeBetweenSendsInSec, } return e, nil @@ -569,30 +572,31 @@ func (e *exportHandlerFactory) prepareFolders(folder string) error { func (e *exportHandlerFactory) createInterceptors() error { argsInterceptors := ArgsNewFullSyncInterceptorsContainerFactory{ - CoreComponents: e.coreComponents, - CryptoComponents: e.cryptoComponents, - Accounts: e.accounts, - ShardCoordinator: e.shardCoordinator, - NodesCoordinator: e.nodesCoordinator, - MainMessenger: e.networkComponents.NetworkMessenger(), - FullArchiveMessenger: e.networkComponents.FullArchiveNetworkMessenger(), - Store: e.storageService, - DataPool: e.dataPool, - MaxTxNonceDeltaAllowed: math.MaxInt32, - TxFeeHandler: &disabled.FeeHandler{}, - BlockBlackList: cache.NewTimeCache(time.Second), - HeaderSigVerifier: e.headerSigVerifier, - HeaderIntegrityVerifier: e.headerIntegrityVerifier, - SizeCheckDelta: math.MaxUint32, - ValidityAttester: e.validityAttester, - EpochStartTrigger: e.epochStartTrigger, - WhiteListHandler: e.whiteListHandler, - WhiteListerVerifiedTxs: e.whiteListerVerifiedTxs, - MainInterceptorsContainer: e.mainInterceptorsContainer, - FullArchiveInterceptorsContainer: e.fullArchiveInterceptorsContainer, - AntifloodHandler: e.networkComponents.InputAntiFloodHandler(), - NodeOperationMode: e.nodeOperationMode, - InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, + CoreComponents: e.coreComponents, + CryptoComponents: e.cryptoComponents, + Accounts: e.accounts, + ShardCoordinator: e.shardCoordinator, + NodesCoordinator: e.nodesCoordinator, + MainMessenger: e.networkComponents.NetworkMessenger(), + FullArchiveMessenger: e.networkComponents.FullArchiveNetworkMessenger(), + Store: e.storageService, + DataPool: e.dataPool, + MaxTxNonceDeltaAllowed: math.MaxInt32, + TxFeeHandler: &disabled.FeeHandler{}, + BlockBlackList: cache.NewTimeCache(time.Second), + HeaderSigVerifier: e.headerSigVerifier, + HeaderIntegrityVerifier: e.headerIntegrityVerifier, + SizeCheckDelta: math.MaxUint32, + ValidityAttester: e.validityAttester, + EpochStartTrigger: e.epochStartTrigger, + WhiteListHandler: e.whiteListHandler, + WhiteListerVerifiedTxs: e.whiteListerVerifiedTxs, + MainInterceptorsContainer: e.mainInterceptorsContainer, + FullArchiveInterceptorsContainer: e.fullArchiveInterceptorsContainer, + AntifloodHandler: e.networkComponents.InputAntiFloodHandler(), + NodeOperationMode: e.nodeOperationMode, + InterceptedDataVerifierFactory: e.interceptedDataVerifierFactory, + PeerAuthenticationTimeBetweenSendsInSec: e.peerAuthenticationTimeBetweenSendsInSec, } fullSyncInterceptors, err := NewFullSyncInterceptorsContainerFactory(argsInterceptors) if err != nil { diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index b81c2b2b393..e2c2300a53d 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -7,7 +7,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" @@ -54,30 +53,31 @@ type fullSyncInterceptorsContainerFactory struct { // ArgsNewFullSyncInterceptorsContainerFactory holds the arguments needed for fullSyncInterceptorsContainerFactory type ArgsNewFullSyncInterceptorsContainerFactory struct { - CoreComponents process.CoreComponentsHolder - CryptoComponents process.CryptoComponentsHolder - Accounts state.AccountsAdapter - ShardCoordinator sharding.Coordinator - NodesCoordinator nodesCoordinator.NodesCoordinator - MainMessenger process.TopicHandler - FullArchiveMessenger process.TopicHandler - Store dataRetriever.StorageService - DataPool dataRetriever.PoolsHolder - MaxTxNonceDeltaAllowed int - TxFeeHandler process.FeeHandler - BlockBlackList process.TimeCacher - HeaderSigVerifier process.InterceptedHeaderSigVerifier - HeaderIntegrityVerifier process.HeaderIntegrityVerifier - SizeCheckDelta uint32 - ValidityAttester process.ValidityAttester - EpochStartTrigger process.EpochStartTriggerHandler - WhiteListHandler update.WhiteListHandler - WhiteListerVerifiedTxs update.WhiteListHandler - MainInterceptorsContainer process.InterceptorsContainer - FullArchiveInterceptorsContainer process.InterceptorsContainer - AntifloodHandler process.P2PAntifloodHandler - NodeOperationMode common.NodeOperation - InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + CoreComponents process.CoreComponentsHolder + CryptoComponents process.CryptoComponentsHolder + Accounts state.AccountsAdapter + ShardCoordinator sharding.Coordinator + NodesCoordinator nodesCoordinator.NodesCoordinator + MainMessenger process.TopicHandler + FullArchiveMessenger process.TopicHandler + Store dataRetriever.StorageService + DataPool dataRetriever.PoolsHolder + MaxTxNonceDeltaAllowed int + TxFeeHandler process.FeeHandler + BlockBlackList process.TimeCacher + HeaderSigVerifier process.InterceptedHeaderSigVerifier + HeaderIntegrityVerifier process.HeaderIntegrityVerifier + SizeCheckDelta uint32 + ValidityAttester process.ValidityAttester + EpochStartTrigger process.EpochStartTriggerHandler + WhiteListHandler update.WhiteListHandler + WhiteListerVerifiedTxs update.WhiteListHandler + MainInterceptorsContainer process.InterceptorsContainer + FullArchiveInterceptorsContainer process.InterceptorsContainer + AntifloodHandler process.P2PAntifloodHandler + NodeOperationMode common.NodeOperation + InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory + PeerAuthenticationTimeBetweenSendsInSec int64 } // NewFullSyncInterceptorsContainerFactory is responsible for creating a new interceptors factory object @@ -140,17 +140,19 @@ func NewFullSyncInterceptorsContainerFactory( } argInterceptorFactory := &interceptorFactory.ArgInterceptedDataFactory{ - CoreComponents: args.CoreComponents, - CryptoComponents: args.CryptoComponents, - ShardCoordinator: args.ShardCoordinator, - NodesCoordinator: args.NodesCoordinator, - FeeHandler: args.TxFeeHandler, - HeaderSigVerifier: args.HeaderSigVerifier, - HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, - ValidityAttester: args.ValidityAttester, - EpochStartTrigger: args.EpochStartTrigger, - WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, - ArgsParser: smartContract.NewArgumentParser(), + CoreComponents: args.CoreComponents, + CryptoComponents: args.CryptoComponents, + ShardCoordinator: args.ShardCoordinator, + NodesCoordinator: args.NodesCoordinator, + FeeHandler: args.TxFeeHandler, + HeaderSigVerifier: args.HeaderSigVerifier, + HeaderIntegrityVerifier: args.HeaderIntegrityVerifier, + ValidityAttester: args.ValidityAttester, + EpochStartTrigger: args.EpochStartTrigger, + WhiteListerVerifiedTxs: args.WhiteListerVerifiedTxs, + ArgsParser: smartContract.NewArgumentParser(), + PeerAuthCacher: args.DataPool.PeerAuthentications(), + PeerAuthenticationTimeBetweenSendsInSec: args.PeerAuthenticationTimeBetweenSendsInSec, } icf := &fullSyncInterceptorsContainerFactory{ From d2bf7675806eec1873b2fe5fd7e06372b8143e67 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Fri, 22 May 2026 15:23:55 +0300 Subject: [PATCH 078/116] fixes after review --- epochStart/bootstrap/process.go | 2 +- .../interceptedPeerAuthentication.go | 98 +++++++++++-------- .../interceptedPeerAuthentication_test.go | 12 ++- ...nterceptedPeerAuthenticationDataFactory.go | 2 +- 4 files changed, 68 insertions(+), 46 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 8d06508dc7e..b6da12f5aa8 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -1467,7 +1467,7 @@ func (e *epochStartBootstrap) createResolversContainer() error { storageService := disabled.NewChainStorer() - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(e.generalConfig.HeartbeatV2.HeartbeatExpiryTimespanInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(e.generalConfig.HeartbeatV2.PeerAuthenticationTimeBetweenChecksInSec) if err != nil { return err } diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 1606d8dea8b..cfa24e8b92d 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -36,19 +36,19 @@ type ArgInterceptedPeerAuthentication struct { // interceptedPeerAuthentication is a wrapper over PeerAuthentication type interceptedPeerAuthentication struct { - peerAuthentication heartbeat.PeerAuthentication - payload heartbeat.Payload - peerId core.PeerID - nodesCoordinator NodesCoordinator - signaturesHandler SignaturesHandler - peerSignatureHandler crypto.PeerSignatureHandler - payloadValidator process.PeerAuthenticationPayloadValidator - hardforkTriggerPubKey []byte - peerShardMapper process.PeerShardMapper - peerAuthCacher storage.Cacher - messageOriginator core.PeerID - selfPeerID core.PeerID - peerAuthenticationTimeBetweenSendsInSec int64 + peerAuthentication heartbeat.PeerAuthentication + payload heartbeat.Payload + peerId core.PeerID + nodesCoordinator NodesCoordinator + signaturesHandler SignaturesHandler + peerSignatureHandler crypto.PeerSignatureHandler + payloadValidator process.PeerAuthenticationPayloadValidator + hardforkTriggerPubKey []byte + peerShardMapper process.PeerShardMapper + peerAuthCacher storage.Cacher + messageOriginator core.PeerID + selfPeerID core.PeerID + peerAuthTimestampDelta int64 } // NewInterceptedPeerAuthentication tries to create a new intercepted peer authentication instance @@ -63,19 +63,21 @@ func NewInterceptedPeerAuthentication(arg ArgInterceptedPeerAuthentication) (*in return nil, err } + deltaSec := float64(arg.PeerAuthenticationTimeBetweenSendsInSec) * maxPercentAllowed + intercepted := &interceptedPeerAuthentication{ - peerAuthentication: *peerAuthentication, - payload: *payload, - nodesCoordinator: arg.NodesCoordinator, - signaturesHandler: arg.SignaturesHandler, - peerSignatureHandler: arg.PeerSignatureHandler, - payloadValidator: arg.PayloadValidator, - hardforkTriggerPubKey: arg.HardforkTriggerPubKey, - peerShardMapper: arg.PeerShardMapper, - peerAuthCacher: arg.PeerAuthCacher, - messageOriginator: arg.MessageOriginator, - selfPeerID: arg.SelfPeerID, - peerAuthenticationTimeBetweenSendsInSec: arg.PeerAuthenticationTimeBetweenSendsInSec, + peerAuthentication: *peerAuthentication, + payload: *payload, + nodesCoordinator: arg.NodesCoordinator, + signaturesHandler: arg.SignaturesHandler, + peerSignatureHandler: arg.PeerSignatureHandler, + payloadValidator: arg.PayloadValidator, + hardforkTriggerPubKey: arg.HardforkTriggerPubKey, + peerShardMapper: arg.PeerShardMapper, + peerAuthCacher: arg.PeerAuthCacher, + messageOriginator: arg.MessageOriginator, + selfPeerID: arg.SelfPeerID, + peerAuthTimestampDelta: int64(deltaSec), } intercepted.peerId = core.PeerID(intercepted.peerAuthentication.Pid) @@ -164,22 +166,12 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { return err } - // Early exit if mapping already exists and the message is from itself - existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) - hasInCache := ipa.peerAuthCacher.Has(ipa.Pubkey()) - isFromSelf := ipa.messageOriginator == ipa.selfPeerID - pairExists := string(existingInfo.PkBytes) == string(ipa.Pubkey()) - if pairExists && isFromSelf { - return nil + shouldSkipSigChecks, errCheck := ipa.checkExistingInfo() + if errCheck != nil { + return errCheck } - if pairExists && !isFromSelf && hasInCache { - return process.ErrPeerAlreadyAuthenticated - } - - deltaSec := float64(ipa.peerAuthenticationTimeBetweenSendsInSec) * maxPercentAllowed - minTimestampAccepted := existingInfo.AuthTimestamp + int64(deltaSec) - if ipa.payload.Timestamp <= minTimestampAccepted { - return fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthTimestamp) + if shouldSkipSigChecks { + return nil } } @@ -206,6 +198,32 @@ func (ipa *interceptedPeerAuthentication) CheckValidity() error { return nil } +func (ipa *interceptedPeerAuthentication) checkExistingInfo() (bool, error) { + existingInfo := ipa.peerShardMapper.GetPeerInfo(ipa.peerId) + pairExists := string(existingInfo.PkBytes) == string(ipa.Pubkey()) + if !pairExists { + return false, nil // continue verification and eventually save in cache + } + + isFromSelf := ipa.messageOriginator == ipa.selfPeerID + if isFromSelf { + return true, nil // skip sig checks + } + + hasInCache := ipa.peerAuthCacher.Has(ipa.Pubkey()) + if !hasInCache { + return false, nil // continue verification and eventually save in cache + } + + minTimestampAccepted := existingInfo.AuthTimestamp + ipa.peerAuthTimestampDelta + if ipa.payload.Timestamp <= minTimestampAccepted { + return false, fmt.Errorf("%w, received timestamp %d while the last one saved is %d", process.ErrPeerAlreadyAuthenticated, ipa.payload.Timestamp, existingInfo.AuthTimestamp) + } + + // mapping exists and delta condition is not satisfied, valid message + return false, nil +} + // IsForCurrentShard always returns true func (ipa *interceptedPeerAuthentication) IsForCurrentShard() bool { return true diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 606e44cc973..3d998b3c697 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -315,31 +315,35 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.NoError(t, err) }) - t.Run("peer already authenticated with newer timestamp should early exit", func(t *testing.T) { + t.Run("peer already authenticated with newer timestamp should early verify", func(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() arg := createMockInterceptedPeerAuthenticationArg(providedPA) + arg.SelfPeerID = "self" authTimestamp := time.Now().Add(time.Minute).Unix() + wasVerifyCalled := false arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { - require.Fail(t, "should have not been called") - return expectedErr + wasVerifyCalled = true + return nil }, } arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { return core.P2PPeerInfo{ AuthTimestamp: authTimestamp, + PkBytes: providedPA.Pubkey, } }, } ipa, _ := NewInterceptedPeerAuthentication(arg) err := ipa.CheckValidity() - assert.ErrorIs(t, err, process.ErrPeerAlreadyAuthenticated) + assert.NoError(t, err) + assert.True(t, wasVerifyCalled) }) t.Run("should work", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index 6985d4a8a96..11385a14aba 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -34,7 +34,7 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) return nil, err } - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(arg.HeartbeatExpiryTimespanInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(arg.PeerAuthenticationTimeBetweenSendsInSec) if err != nil { return nil, err } From ab7bf5ae396629b212a88e8fb66a2e66ce4e85bc Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Fri, 22 May 2026 16:06:01 +0300 Subject: [PATCH 079/116] fix tests after fixes after review --- epochStart/bootstrap/process.go | 2 +- factory/heartbeat/heartbeatV2Components_test.go | 2 +- factory/processing/processComponents.go | 2 +- integrationTests/testProcessorNode.go | 2 +- testscommon/generalConfig.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index b6da12f5aa8..f4d2526cd3b 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -1467,7 +1467,7 @@ func (e *epochStartBootstrap) createResolversContainer() error { storageService := disabled.NewChainStorer() - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(e.generalConfig.HeartbeatV2.PeerAuthenticationTimeBetweenChecksInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(e.generalConfig.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec) if err != nil { return err } diff --git a/factory/heartbeat/heartbeatV2Components_test.go b/factory/heartbeat/heartbeatV2Components_test.go index f605bc67b9c..b6e3ca53975 100644 --- a/factory/heartbeat/heartbeatV2Components_test.go +++ b/factory/heartbeat/heartbeatV2Components_test.go @@ -93,7 +93,7 @@ func createMockHeartbeatV2ComponentsFactoryArgs() heartbeatComp.ArgHeartbeatV2Co func createMockConfig() config.Config { return config.Config{ HeartbeatV2: config.HeartbeatV2Config{ - PeerAuthenticationTimeBetweenSendsInSec: 1, + PeerAuthenticationTimeBetweenSendsInSec: 10, PeerAuthenticationTimeBetweenSendsWhenErrorInSec: 1, PeerAuthenticationTimeThresholdBetweenSends: 0.1, HeartbeatTimeBetweenSendsInSec: 1, diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 066efb32d21..1e5cc724da5 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1395,7 +1395,7 @@ func (pcf *processComponentsFactory) newResolverContainerFactory() (dataRetrieve return disabledResolversContainer.NewDisabledResolversContainerFactory(), nil } - payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(pcf.config.HeartbeatV2.HeartbeatExpiryTimespanInSec) + payloadValidator, err := validator.NewPeerAuthenticationPayloadValidator(pcf.config.HeartbeatV2.PeerAuthenticationTimeBetweenSendsInSec) if err != nil { return nil, err } diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index ea84784dce1..e6b99421a21 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -3277,7 +3277,7 @@ func (tpn *TestProcessorNode) createHeartbeatWithHardforkTrigger() { // ============== HeartbeatV2 ============= // hbv2Config := config.HeartbeatV2Config{ - PeerAuthenticationTimeBetweenSendsInSec: 5, + PeerAuthenticationTimeBetweenSendsInSec: 10, PeerAuthenticationTimeBetweenSendsWhenErrorInSec: 1, PeerAuthenticationTimeThresholdBetweenSends: 0.1, HeartbeatTimeBetweenSendsInSec: 2, diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index 5153b670970..bb7731223d6 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -262,7 +262,7 @@ func GetGeneralConfig() config.Config { }, }, HeartbeatV2: config.HeartbeatV2Config{ - PeerAuthenticationTimeBetweenSendsInSec: 1, + PeerAuthenticationTimeBetweenSendsInSec: 10, PeerAuthenticationTimeBetweenSendsWhenErrorInSec: 1, PeerAuthenticationTimeThresholdBetweenSends: 0.1, HeartbeatTimeBetweenSendsInSec: 1, From a061750702e63c5484ae044172dd03e6cada88fe Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 25 May 2026 13:00:08 +0300 Subject: [PATCH 080/116] fix silent encode --- .../transactionAPI/apiTransactionProcessor.go | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/node/external/transactionAPI/apiTransactionProcessor.go b/node/external/transactionAPI/apiTransactionProcessor.go index 5f5c47a50cc..ea9af5ed08b 100644 --- a/node/external/transactionAPI/apiTransactionProcessor.go +++ b/node/external/transactionAPI/apiTransactionProcessor.go @@ -267,7 +267,7 @@ func (atp *apiTransactionProcessor) GetTransactionsPoolForSender(sender, fields requestedFieldsHandler := newFieldsHandler(fields) transactions := &common.TransactionsPoolForSenderApiResponse{} for _, wrappedTx := range wrappedTxs { - tx := atp.extractRequestedTxInfo(wrappedTx, requestedFieldsHandler) + tx := atp.extractRequestedTxInfo(wrappedTx, requestedFieldsHandler, transaction.TxTypeNormal) // use TxTypeNormal for all, only used for sender encoding transactions.Transactions = append(transactions.Transactions, tx) } @@ -318,7 +318,7 @@ func (atp *apiTransactionProcessor) extractRequestedTxInfoFromObj(txObj interfac TxHash: txHash, } - requestedTxInfo := atp.extractRequestedTxInfo(wrappedTx, requestedFieldsHandler) + requestedTxInfo := atp.extractRequestedTxInfo(wrappedTx, requestedFieldsHandler, txType) return requestedTxInfo } @@ -374,8 +374,12 @@ func (atp *apiTransactionProcessor) getUnsignedTransactionsFromPool(requestedFie return unsignedTxs } -func (atp *apiTransactionProcessor) extractRequestedTxInfo(wrappedTx *txcache.WrappedTransaction, requestedFieldsHandler fieldsHandler) common.Transaction { - fieldGetters := atp.getFieldGettersForTx(wrappedTx) +func (atp *apiTransactionProcessor) extractRequestedTxInfo( + wrappedTx *txcache.WrappedTransaction, + requestedFieldsHandler fieldsHandler, + txType transaction.TxType, +) common.Transaction { + fieldGetters := atp.getFieldGettersForTx(wrappedTx, txType) tx := common.Transaction{ TxFields: make(map[string]interface{}), } @@ -389,11 +393,19 @@ func (atp *apiTransactionProcessor) extractRequestedTxInfo(wrappedTx *txcache.Wr return tx } -func (atp *apiTransactionProcessor) getFieldGettersForTx(wrappedTx *txcache.WrappedTransaction) map[string]interface{} { +func (atp *apiTransactionProcessor) getFieldGettersForTx( + wrappedTx *txcache.WrappedTransaction, + txType transaction.TxType, +) map[string]interface{} { + senderStr := "metachain" + if txType != transaction.TxTypeReward { + senderStr = atp.addressPubKeyConverter.SilentEncode(wrappedTx.Tx.GetSndAddr(), log) + } + var fieldGetters = map[string]interface{}{ hashField: hex.EncodeToString(wrappedTx.TxHash), nonceField: wrappedTx.Tx.GetNonce(), - senderField: atp.addressPubKeyConverter.SilentEncode(wrappedTx.Tx.GetSndAddr(), log), + senderField: senderStr, receiverField: atp.addressPubKeyConverter.SilentEncode(wrappedTx.Tx.GetRcvAddr(), log), gasLimitField: wrappedTx.Tx.GetGasLimit(), gasPriceField: wrappedTx.Tx.GetGasPrice(), From dc691833aa7b6f3cd63fde39bc5901063616521f Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 25 May 2026 14:22:18 +0300 Subject: [PATCH 081/116] fix fallback shard --- epochStart/bootstrap/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index f4d2526cd3b..b4f2dd14cba 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -361,7 +361,7 @@ func (e *epochStartBootstrap) Bootstrap() (Parameters, error) { newShardId, _, err := e.getShardIDForLatestEpoch() if err != nil { // fallback to meta if nothing was loaded from the last epoch - newShardId = core.MetachainShardId + newShardId = e.applyShardIDAsObserverIfNeeded(core.MetachainShardId) } log.Debug("epochStartBootstrap.Bootstrap", "newShardId", newShardId, "from last epoch", err == nil) From 1d3fe6efdce5fd51a51afa49b928fc8190caf1c9 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 26 May 2026 14:00:05 +0300 Subject: [PATCH 082/116] use deadline on direct send --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 98247418f4f..f9362d10b25 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/libp2p/go-libp2p v0.38.2 github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-communication-go v1.3.0 + github.com/multiversx/mx-chain-communication-go v1.3.2-0.20260526105332-149c7edf8889 github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 github.com/multiversx/mx-chain-crypto-go v1.3.0 github.com/multiversx/mx-chain-es-indexer-go v1.9.3 diff --git a/go.sum b/go.sum index 2341bedfa9d..f7d505fd8d3 100644 --- a/go.sum +++ b/go.sum @@ -399,8 +399,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= -github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= +github.com/multiversx/mx-chain-communication-go v1.3.2-0.20260526105332-149c7edf8889 h1:HyqgosxMyWvNnGMXSubqARJ+DFP9O8BR2aczUg6v/gs= +github.com/multiversx/mx-chain-communication-go v1.3.2-0.20260526105332-149c7edf8889/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 h1:hpnYOT5cDJip7B6GvFRSOcUdwdhBbuFsNjLvOxzYOn8= github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= From 0f82bf1a0ee0ce500ac70f0b3f01d5ba7649fb6d Mon Sep 17 00:00:00 2001 From: radu Date: Tue, 26 May 2026 19:09:22 +0300 Subject: [PATCH 083/116] added peerAuthentication identifiers to whiteList --- .../requestHandlers/requestHandler.go | 8 +++++ .../requestHandlers/requestHandler_test.go | 15 ++++++-- p2p/constants.go | 3 ++ .../interceptedPeerAuthentication.go | 4 ++- .../interceptedPeerAuthentication_test.go | 6 +++- process/interceptors/multiDataInterceptor.go | 35 +++++++++++-------- 6 files changed, 51 insertions(+), 20 deletions(-) diff --git a/dataRetriever/requestHandlers/requestHandler.go b/dataRetriever/requestHandlers/requestHandler.go index 05f8152ecf3..f44a4c8f8b4 100644 --- a/dataRetriever/requestHandlers/requestHandler.go +++ b/dataRetriever/requestHandlers/requestHandler.go @@ -870,6 +870,14 @@ func (rrh *resolverRequestHandler) RequestPeerAuthenticationsByHashes(destShardI return } + identifiers := make([][]byte, 0, len(hashes)) + for _, hash := range hashes { + identifier := common.PeerAuthenticationPublicKeyIdentifier(hash) + identifiers = append(identifiers, identifier) + } + + rrh.whiteList.Add(identifiers) + err = peerAuthRequester.RequestDataFromHashArray(hashes, epoch) if err != nil { log.Debug("RequestPeerAuthenticationsByHashes.RequestDataFromHashArray", diff --git a/dataRetriever/requestHandlers/requestHandler_test.go b/dataRetriever/requestHandlers/requestHandler_test.go index f4922d5d5b7..d9048a7f410 100644 --- a/dataRetriever/requestHandlers/requestHandler_test.go +++ b/dataRetriever/requestHandlers/requestHandler_test.go @@ -1578,10 +1578,13 @@ func TestResolverRequestHandler_RequestPeerAuthenticationsByHashes(t *testing.T) }() wasCalled := false + wasWhitelisted := false + longPublicKey := bytes.Repeat([]byte("p"), common.MaxPeerAuthenticationPublicKeyIdentifierLen+8) + providedHashesForTest := [][]byte{longPublicKey, []byte("h2")} paRequester := &dataRetrieverMocks.HashSliceRequesterStub{ RequestDataFromHashArrayCalled: func(hashes [][]byte, epoch uint32) error { wasCalled = true - assert.Equal(t, providedHashes, hashes) + assert.Equal(t, providedHashesForTest, hashes) return nil }, } @@ -1593,15 +1596,21 @@ func TestResolverRequestHandler_RequestPeerAuthenticationsByHashes(t *testing.T) }, }, &mock.RequestedItemsHandlerStub{}, - &mock.WhiteListHandlerStub{}, + &mock.WhiteListHandlerStub{ + AddCalled: func(keys [][]byte) { + wasWhitelisted = true + assert.Equal(t, [][]byte{longPublicKey[:common.MaxPeerAuthenticationPublicKeyIdentifierLen], []byte("h2")}, keys) + }, + }, 1, 0, time.Second, time.Millisecond, ) - rrh.RequestPeerAuthenticationsByHashes(providedShardId, providedHashes) + rrh.RequestPeerAuthenticationsByHashes(providedShardId, providedHashesForTest) assert.True(t, wasCalled) + assert.True(t, wasWhitelisted) }) } diff --git a/p2p/constants.go b/p2p/constants.go index 8a6db9caeb4..b99373cea2f 100644 --- a/p2p/constants.go +++ b/p2p/constants.go @@ -30,3 +30,6 @@ const DefaultWithScaleResourceLimiter = p2p.DefaultWithScaleResourceLimiter // BroadcastMethod defines the broadcast method of the message type BroadcastMethod = p2p.BroadcastMethod + +// Direct defines a direct message +const Direct = p2p.Direct diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index cfa24e8b92d..824fa661d3d 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/marshal" crypto "github.com/multiversx/mx-chain-crypto-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" @@ -241,7 +242,8 @@ func (ipa *interceptedPeerAuthentication) Type() string { // Identifiers returns the identifiers used in requests func (ipa *interceptedPeerAuthentication) Identifiers() [][]byte { - return [][]byte{ipa.peerAuthentication.Pubkey, ipa.peerAuthentication.Pid} + identifier := common.PeerAuthenticationPublicKeyIdentifier(ipa.peerAuthentication.Pubkey) + return [][]byte{ipa.peerAuthentication.Pubkey, ipa.peerAuthentication.Pid, identifier} } // PeerID returns the peer ID diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 3d998b3c697..933a64c1774 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" @@ -410,6 +411,8 @@ func TestInterceptedPeerAuthentication_Getters(t *testing.T) { t.Parallel() providedPA := createDefaultInterceptedPeerAuthentication() + require.NotNil(t, providedPA) + providedPA.Pubkey = []byte("0123456789012345678901234567890123456789") arg := createMockInterceptedPeerAuthenticationArg(providedPA) ipa, _ := NewInterceptedPeerAuthentication(arg) expectedPeerAuthentication := &heartbeat.PeerAuthentication{} @@ -425,9 +428,10 @@ func TestInterceptedPeerAuthentication_Getters(t *testing.T) { assert.Equal(t, expectedPeerAuthentication.Pubkey, ipa.Pubkey()) identifiers := ipa.Identifiers() - assert.Equal(t, 2, len(identifiers)) + assert.Equal(t, 3, len(identifiers)) assert.Equal(t, expectedPeerAuthentication.Pubkey, identifiers[0]) assert.Equal(t, expectedPeerAuthentication.Pid, identifiers[1]) + assert.Equal(t, expectedPeerAuthentication.Pubkey[:common.MaxPeerAuthenticationPublicKeyIdentifierLen], identifiers[2]) providedPASize := getSizeOfPA(providedPA) assert.Equal(t, providedPASize, ipa.SizeInBytes()) } diff --git a/process/interceptors/multiDataInterceptor.go b/process/interceptors/multiDataInterceptor.go index 054a57a0dbe..5983d06f756 100644 --- a/process/interceptors/multiDataInterceptor.go +++ b/process/interceptors/multiDataInterceptor.go @@ -160,20 +160,25 @@ func (mdi *MultiDataInterceptor) ProcessReceivedMessage(message p2p.MessageP2P, multiDataBuff = [][]byte{checkChunksRes.CompleteBuffer} } - listInterceptedData := make([]process.InterceptedData, len(multiDataBuff)) + listInterceptedData := make([]process.InterceptedData, 0, len(multiDataBuff)) errOriginator := mdi.antifloodHandler.IsOriginatorEligibleForTopic(message.Peer(), mdi.topic) - - for index, dataBuff := range multiDataBuff { + var isWhiteListed bool + for _, dataBuff := range multiDataBuff { var interceptedData process.InterceptedData - interceptedData, err = mdi.interceptedData(dataBuff, message, fromConnectedPeer, errOriginator) - listInterceptedData[index] = interceptedData + interceptedData, isWhiteListed, err = mdi.interceptedData(dataBuff, message, fromConnectedPeer, errOriginator) - if err != nil { - mdi.throttler.EndProcessing() - return nil, err + if err == nil { + listInterceptedData = append(listInterceptedData, interceptedData) + continue } - } + if isWhiteListed && message.BroadcastMethod() == p2p.Direct { + continue + } + + mdi.throttler.EndProcessing() + return nil, err + } go func() { for _, interceptedData := range listInterceptedData { mdi.processInterceptedData(interceptedData, message) @@ -211,7 +216,7 @@ func (mdi *MultiDataInterceptor) interceptedData( message p2p.MessageP2P, fromConnectedPeer core.PeerID, errOriginator error, -) (process.InterceptedData, error) { +) (process.InterceptedData, bool, error) { originator := message.Peer() interceptedData, err := mdi.factory.Create(dataBuff, originator) if err != nil { @@ -220,7 +225,7 @@ func (mdi *MultiDataInterceptor) interceptedData( mdi.antifloodHandler.BlacklistPeer(originator, reason, common.InvalidMessageBlacklistDuration) mdi.antifloodHandler.BlacklistPeer(fromConnectedPeer, reason, common.InvalidMessageBlacklistDuration) - return nil, err + return nil, false, err } mdi.receivedDebugInterceptedData(interceptedData) @@ -231,7 +236,7 @@ func (mdi *MultiDataInterceptor) interceptedData( p2p.PeerIdToShortString(originator), "topic", mdi.topic, "err", errOriginator) - return nil, errOriginator + return nil, isWhiteListed, errOriginator } isForCurrentShard := interceptedData.IsForCurrentShard() @@ -243,7 +248,7 @@ func (mdi *MultiDataInterceptor) interceptedData( "hash", interceptedData.Hash(), "is for this shard", isForCurrentShard, ) - return nil, process.ErrInterceptedDataNotForCurrentShard + return nil, isWhiteListed, process.ErrInterceptedDataNotForCurrentShard } err = mdi.interceptedDataVerifier.Verify(interceptedData) @@ -258,10 +263,10 @@ func (mdi *MultiDataInterceptor) interceptedData( mdi.antifloodHandler.BlacklistPeer(fromConnectedPeer, reason, common.InvalidMessageBlacklistDuration) } - return nil, err + return nil, isWhiteListed, err } - return interceptedData, nil + return interceptedData, isWhiteListed, nil } // RegisterHandler registers a callback function to be notified on received data From 867c06b1842a9a202449e9899ffb6c19d73b3732 Mon Sep 17 00:00:00 2001 From: radu Date: Tue, 26 May 2026 19:20:07 +0300 Subject: [PATCH 084/116] added peer authentication identifier trim method to common --- common/peerAuthentication.go | 14 +++++++++++++ common/peerAuthentication_test.go | 34 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 common/peerAuthentication.go create mode 100644 common/peerAuthentication_test.go diff --git a/common/peerAuthentication.go b/common/peerAuthentication.go new file mode 100644 index 00000000000..49132e9b32e --- /dev/null +++ b/common/peerAuthentication.go @@ -0,0 +1,14 @@ +package common + +// MaxPeerAuthenticationPublicKeyIdentifierLen is the maximum number of public key bytes used as +// a peer authentication request/whitelist identifier. +const MaxPeerAuthenticationPublicKeyIdentifierLen = 32 + +// PeerAuthenticationPublicKeyIdentifier returns the public key prefix used as request/whitelist identifier. +func PeerAuthenticationPublicKeyIdentifier(publicKey []byte) []byte { + if len(publicKey) > MaxPeerAuthenticationPublicKeyIdentifierLen { + publicKey = publicKey[:MaxPeerAuthenticationPublicKeyIdentifierLen] + } + + return append([]byte(nil), publicKey...) +} diff --git a/common/peerAuthentication_test.go b/common/peerAuthentication_test.go new file mode 100644 index 00000000000..7b23145c642 --- /dev/null +++ b/common/peerAuthentication_test.go @@ -0,0 +1,34 @@ +package common_test + +import ( + "bytes" + "testing" + + "github.com/multiversx/mx-chain-go/common" + "github.com/stretchr/testify/require" +) + +func TestPeerAuthenticationPublicKeyIdentifier(t *testing.T) { + t.Parallel() + + t.Run("short public key should be copied unchanged", func(t *testing.T) { + t.Parallel() + + publicKey := []byte("public key") + + identifier := common.PeerAuthenticationPublicKeyIdentifier(publicKey) + + require.Equal(t, publicKey, identifier) + require.NotSame(t, &publicKey[0], &identifier[0]) + }) + t.Run("long public key should be trimmed and copied", func(t *testing.T) { + t.Parallel() + + publicKey := bytes.Repeat([]byte("p"), common.MaxPeerAuthenticationPublicKeyIdentifierLen+8) + + identifier := common.PeerAuthenticationPublicKeyIdentifier(publicKey) + + require.Equal(t, publicKey[:common.MaxPeerAuthenticationPublicKeyIdentifierLen], identifier) + require.NotSame(t, &publicKey[0], &identifier[0]) + }) +} From b644862a45524b4495e1ed17c6c26d19612e234c Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 12:25:30 +0300 Subject: [PATCH 085/116] updated core-go and communication-go tags --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f9362d10b25..f1a9313fa6c 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/libp2p/go-libp2p v0.38.2 github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-communication-go v1.3.2-0.20260526105332-149c7edf8889 - github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 + github.com/multiversx/mx-chain-communication-go v1.3.2 + github.com/multiversx/mx-chain-core-go v1.4.2 github.com/multiversx/mx-chain-crypto-go v1.3.0 github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 diff --git a/go.sum b/go.sum index f7d505fd8d3..c9d06a55d01 100644 --- a/go.sum +++ b/go.sum @@ -399,10 +399,10 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.3.2-0.20260526105332-149c7edf8889 h1:HyqgosxMyWvNnGMXSubqARJ+DFP9O8BR2aczUg6v/gs= -github.com/multiversx/mx-chain-communication-go v1.3.2-0.20260526105332-149c7edf8889/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62 h1:hpnYOT5cDJip7B6GvFRSOcUdwdhBbuFsNjLvOxzYOn8= -github.com/multiversx/mx-chain-core-go v1.4.2-0.20260505075936-43445d8a0f62/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-communication-go v1.3.2 h1:LXz6mjo9hb9X9BIcMFWgJlDVhG8uB2GQWzefSwQAHc0= +github.com/multiversx/mx-chain-communication-go v1.3.2/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= +github.com/multiversx/mx-chain-core-go v1.4.2 h1:/2I0ldOc9Y6zWSG1qzJ9nfdEKk5Spwhi2fKWv0tCFMU= +github.com/multiversx/mx-chain-core-go v1.4.2/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= github.com/multiversx/mx-chain-es-indexer-go v1.9.3 h1:mtc4jxbFoURpF+UmOjD1/cc4XBGh4WyKGduOV4BCGBQ= From ba2f712685a3bf407460c8ec52c07323c33a89a8 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 13:47:45 +0300 Subject: [PATCH 086/116] fix after audit, pass peer id --- .../factory/interceptedPeerAuthenticationDataFactory.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index 11385a14aba..b884553d4bd 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -25,6 +25,7 @@ type interceptedPeerAuthenticationDataFactory struct { peerShardMapper process.PeerShardMapper peerAuthCacher storage.Cacher peerAuthenticationTimeBetweenSendsInSec int64 + selfID core.PeerID } // NewInterceptedPeerAuthenticationDataFactory creates an instance of interceptedPeerAuthenticationDataFactory @@ -49,6 +50,7 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) peerShardMapper: arg.PeerShardMapper, peerAuthCacher: arg.PeerAuthCacher, peerAuthenticationTimeBetweenSendsInSec: arg.PeerAuthenticationTimeBetweenSendsInSec, + selfID: arg.PeerID, }, nil } @@ -79,7 +81,7 @@ func checkArgInterceptedDataFactory(args ArgInterceptedDataFactory) error { } // Create creates instances of InterceptedData by unmarshalling provided buffer -func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ core.PeerID) (process.InterceptedData, error) { +func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, messageOriginator core.PeerID) (process.InterceptedData, error) { arg := heartbeat.ArgInterceptedPeerAuthentication{ ArgBaseInterceptedHeartbeat: heartbeat.ArgBaseInterceptedHeartbeat{ DataBuff: buff, @@ -92,6 +94,8 @@ func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, _ cor HardforkTriggerPubKey: ipadf.hardforkTriggerPubKey, PeerShardMapper: ipadf.peerShardMapper, PeerAuthCacher: ipadf.peerAuthCacher, + MessageOriginator: messageOriginator, + SelfPeerID: ipadf.selfID, PeerAuthenticationTimeBetweenSendsInSec: ipadf.peerAuthenticationTimeBetweenSendsInSec, } From 652c3c8b0b976b827004c95b5167c988c6da568f Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 14:02:21 +0300 Subject: [PATCH 087/116] fix for multikey as well --- process/heartbeat/interceptedPeerAuthentication.go | 9 ++++++++- .../heartbeat/interceptedPeerAuthentication_test.go | 11 +++++++++++ .../interceptors/factory/argInterceptedDataFactory.go | 4 ++-- .../interceptedPeerAuthenticationDataFactory.go | 4 ++++ .../peerAuthenticationInterceptorProcessor_test.go | 1 + 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 824fa661d3d..0f55e89af8d 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -33,6 +33,7 @@ type ArgInterceptedPeerAuthentication struct { MessageOriginator core.PeerID SelfPeerID core.PeerID PeerAuthenticationTimeBetweenSendsInSec int64 + ManagedPeersHolder common.ManagedPeersHolder } // interceptedPeerAuthentication is a wrapper over PeerAuthentication @@ -49,6 +50,7 @@ type interceptedPeerAuthentication struct { peerAuthCacher storage.Cacher messageOriginator core.PeerID selfPeerID core.PeerID + managedPeersHolder common.ManagedPeersHolder peerAuthTimestampDelta int64 } @@ -79,6 +81,7 @@ func NewInterceptedPeerAuthentication(arg ArgInterceptedPeerAuthentication) (*in messageOriginator: arg.MessageOriginator, selfPeerID: arg.SelfPeerID, peerAuthTimestampDelta: int64(deltaSec), + managedPeersHolder: arg.ManagedPeersHolder, } intercepted.peerId = core.PeerID(intercepted.peerAuthentication.Pid) @@ -114,6 +117,9 @@ func checkArg(arg ArgInterceptedPeerAuthentication) error { if arg.PeerAuthenticationTimeBetweenSendsInSec < minPeerAuthenticationTimeBetweenSendsInSec { return fmt.Errorf("%w for PeerAuthenticationTimeBetweenSendsInSec", process.ErrInvalidValue) } + if check.IfNil(arg.ManagedPeersHolder) { + return process.ErrNilManagedPeersHolder + } return nil } @@ -207,7 +213,8 @@ func (ipa *interceptedPeerAuthentication) checkExistingInfo() (bool, error) { } isFromSelf := ipa.messageOriginator == ipa.selfPeerID - if isFromSelf { + isManagedBySelf := ipa.managedPeersHolder.IsPidManagedByCurrentNode(ipa.messageOriginator) + if isFromSelf || isManagedBySelf { return true, nil // skip sig checks } diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index 933a64c1774..fd7520cd0dd 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -65,6 +65,7 @@ func createMockInterceptedPeerAuthenticationArg(interceptedData *heartbeat.PeerA PeerShardMapper: &processMocks.PeerShardMapperStub{}, PeerAuthCacher: cache.NewCacherStub(), PeerAuthenticationTimeBetweenSendsInSec: 10, + ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, } arg.DataBuff, _ = arg.Marshaller.Marshal(interceptedData) @@ -154,6 +155,16 @@ func TestNewInterceptedPeerAuthentication(t *testing.T) { assert.True(t, check.IfNil(ipa)) assert.Equal(t, process.ErrNilPeerAuthenticationCacher, err) }) + t.Run("nil managed peers holder should error", func(t *testing.T) { + t.Parallel() + + arg := createMockInterceptedPeerAuthenticationArg(createDefaultInterceptedPeerAuthentication()) + arg.ManagedPeersHolder = nil + + ipa, err := NewInterceptedPeerAuthentication(arg) + assert.True(t, check.IfNil(ipa)) + assert.Equal(t, process.ErrNilManagedPeersHolder, err) + }) t.Run("invalid peer auth time between sends should error", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/factory/argInterceptedDataFactory.go b/process/interceptors/factory/argInterceptedDataFactory.go index 4f50225091f..c2eb34bdfaa 100644 --- a/process/interceptors/factory/argInterceptedDataFactory.go +++ b/process/interceptors/factory/argInterceptedDataFactory.go @@ -6,12 +6,11 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" crypto "github.com/multiversx/mx-chain-crypto-go" - "github.com/multiversx/mx-chain-go/storage" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" + "github.com/multiversx/mx-chain-go/storage" ) // interceptedDataCoreComponentsHolder holds the core components required by the intercepted data factory @@ -40,6 +39,7 @@ type interceptedDataCryptoComponentsHolder interface { BlockSigner() crypto.SingleSigner GetMultiSigner(epoch uint32) (crypto.MultiSigner, error) PublicKey() crypto.PublicKey + ManagedPeersHolder() common.ManagedPeersHolder IsInterfaceNil() bool } diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index b884553d4bd..c3ed49c3ecf 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/marshal" crypto "github.com/multiversx/mx-chain-crypto-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/heartbeat" "github.com/multiversx/mx-chain-go/process/heartbeat/validator" @@ -26,6 +27,7 @@ type interceptedPeerAuthenticationDataFactory struct { peerAuthCacher storage.Cacher peerAuthenticationTimeBetweenSendsInSec int64 selfID core.PeerID + managedPeersHolder common.ManagedPeersHolder } // NewInterceptedPeerAuthenticationDataFactory creates an instance of interceptedPeerAuthenticationDataFactory @@ -51,6 +53,7 @@ func NewInterceptedPeerAuthenticationDataFactory(arg ArgInterceptedDataFactory) peerAuthCacher: arg.PeerAuthCacher, peerAuthenticationTimeBetweenSendsInSec: arg.PeerAuthenticationTimeBetweenSendsInSec, selfID: arg.PeerID, + managedPeersHolder: arg.CryptoComponents.ManagedPeersHolder(), }, nil } @@ -97,6 +100,7 @@ func (ipadf *interceptedPeerAuthenticationDataFactory) Create(buff []byte, messa MessageOriginator: messageOriginator, SelfPeerID: ipadf.selfID, PeerAuthenticationTimeBetweenSendsInSec: ipadf.peerAuthenticationTimeBetweenSendsInSec, + ManagedPeersHolder: ipadf.managedPeersHolder, } return heartbeat.NewInterceptedPeerAuthentication(arg) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index aa1abb4ed50..1e402d7b220 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -65,6 +65,7 @@ func createMockInterceptedPeerAuthentication() process.InterceptedData { PeerShardMapper: &mock.PeerShardMapperStub{}, PeerAuthCacher: cache.NewCacherStub(), PeerAuthenticationTimeBetweenSendsInSec: 10, + ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, } arg.DataBuff, _ = arg.Marshaller.Marshal(createInterceptedPeerAuthentication()) ipa, _ := heartbeat.NewInterceptedPeerAuthentication(arg) From 5977c7063c50b85f881939654e2447b0f5343126 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 14:10:47 +0300 Subject: [PATCH 088/116] fix tests --- integrationTests/testHeartbeatNode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integrationTests/testHeartbeatNode.go b/integrationTests/testHeartbeatNode.go index 1936b2590d0..1d977806bb1 100644 --- a/integrationTests/testHeartbeatNode.go +++ b/integrationTests/testHeartbeatNode.go @@ -649,6 +649,7 @@ func (thn *TestHeartbeatNode) initInterceptors() { PeerShardMapper: thn.MainPeerShardMapper, PeerAuthCacher: thn.DataPool.PeerAuthentications(), PeerAuthenticationTimeBetweenSendsInSec: thn.heartbeatExpiryTimespanInSec, + CryptoComponents: GetDefaultCryptoComponents(), } thn.createPeerAuthInterceptor(argsFactory) From ed386cb4b26ab273621eb86211a49f965677ea0c Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 14:16:20 +0300 Subject: [PATCH 089/116] fix tests --- .../factory/interceptedMetaHeaderDataFactory_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go index 45e7493f2e3..27e3c5fa22b 100644 --- a/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go +++ b/process/interceptors/factory/interceptedMetaHeaderDataFactory_test.go @@ -89,6 +89,7 @@ func createMockComponentHolders() (*mock.CoreComponentsMock, *mock.CryptoCompone MultiSigContainer: cryptoMocks.NewMultiSignerContainerMock(cryptoMocks.NewMultiSigner()), BlKeyGen: createMockKeyGen(), TxKeyGen: createMockKeyGen(), + ManagedPeers: &testscommon.ManagedPeersHolderStub{}, } return coreComponents, cryptoComponents From df0e2975bb5d67bc01065c6d2ede926b03c725ad Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 14:35:12 +0300 Subject: [PATCH 090/116] fixes after ai review --- .../interceptedPeerAuthentication.go | 7 +++++++ .../interceptedPeerAuthentication_test.go | 21 +++++++++++++++++++ ...nterceptedPeerAuthenticationDataFactory.go | 3 +++ ...eptedPeerAuthenticationDataFactory_test.go | 11 ++++++++++ 4 files changed, 42 insertions(+) diff --git a/process/heartbeat/interceptedPeerAuthentication.go b/process/heartbeat/interceptedPeerAuthentication.go index 0f55e89af8d..9b4512fb528 100644 --- a/process/heartbeat/interceptedPeerAuthentication.go +++ b/process/heartbeat/interceptedPeerAuthentication.go @@ -117,6 +117,9 @@ func checkArg(arg ArgInterceptedPeerAuthentication) error { if arg.PeerAuthenticationTimeBetweenSendsInSec < minPeerAuthenticationTimeBetweenSendsInSec { return fmt.Errorf("%w for PeerAuthenticationTimeBetweenSendsInSec", process.ErrInvalidValue) } + if len(arg.SelfPeerID) == 0 { + return fmt.Errorf("%w for self peer id", process.ErrInvalidValue) + } if check.IfNil(arg.ManagedPeersHolder) { return process.ErrNilManagedPeersHolder } @@ -212,6 +215,10 @@ func (ipa *interceptedPeerAuthentication) checkExistingInfo() (bool, error) { return false, nil // continue verification and eventually save in cache } + if len(ipa.messageOriginator) == 0 { + return false, nil + } + isFromSelf := ipa.messageOriginator == ipa.selfPeerID isManagedBySelf := ipa.managedPeersHolder.IsPidManagedByCurrentNode(ipa.messageOriginator) if isFromSelf || isManagedBySelf { diff --git a/process/heartbeat/interceptedPeerAuthentication_test.go b/process/heartbeat/interceptedPeerAuthentication_test.go index fd7520cd0dd..39d65254be7 100644 --- a/process/heartbeat/interceptedPeerAuthentication_test.go +++ b/process/heartbeat/interceptedPeerAuthentication_test.go @@ -66,6 +66,8 @@ func createMockInterceptedPeerAuthenticationArg(interceptedData *heartbeat.PeerA PeerAuthCacher: cache.NewCacherStub(), PeerAuthenticationTimeBetweenSendsInSec: 10, ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, + SelfPeerID: "self", + MessageOriginator: "originator", } arg.DataBuff, _ = arg.Marshaller.Marshal(interceptedData) @@ -308,6 +310,8 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { providedPA := createDefaultInterceptedPeerAuthentication() arg := createMockInterceptedPeerAuthenticationArg(providedPA) + providedPA.Pid = []byte(arg.SelfPeerID) + arg.MessageOriginator = arg.SelfPeerID arg.SignaturesHandler = &processMocks.SignaturesHandlerStub{ VerifyCalled: func(payload []byte, pid core.PeerID, signature []byte) error { @@ -365,6 +369,23 @@ func TestInterceptedPeerAuthentication_CheckValidity(t *testing.T) { err := ipa.CheckValidity() assert.Nil(t, err) }) + t.Run("should work with empty originator", func(t *testing.T) { + t.Parallel() + + providedPA := createDefaultInterceptedPeerAuthentication() + arg := createMockInterceptedPeerAuthenticationArg(providedPA) + arg.MessageOriginator = "" + arg.PeerShardMapper = &processMocks.PeerShardMapperStub{ + GetPeerInfoCalled: func(pid core.PeerID) core.P2PPeerInfo { + return core.P2PPeerInfo{ + PkBytes: providedPA.Pubkey, + } + }, + } + ipa, _ := NewInterceptedPeerAuthentication(arg) + err := ipa.CheckValidity() + assert.Nil(t, err) + }) t.Run("should work - hardfork from source", func(t *testing.T) { t.Parallel() diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go index c3ed49c3ecf..bd7ef59ec3e 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory.go @@ -64,6 +64,9 @@ func checkArgInterceptedDataFactory(args ArgInterceptedDataFactory) error { if check.IfNil(args.CoreComponents.InternalMarshalizer()) { return process.ErrNilMarshalizer } + if check.IfNil(args.CryptoComponents) { + return process.ErrNilCryptoComponentsHolder + } if check.IfNil(args.NodesCoordinator) { return process.ErrNilNodesCoordinator } diff --git a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory_test.go b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory_test.go index d1de48a25ed..ad83aaea5fd 100644 --- a/process/interceptors/factory/interceptedPeerAuthenticationDataFactory_test.go +++ b/process/interceptors/factory/interceptedPeerAuthenticationDataFactory_test.go @@ -38,6 +38,17 @@ func TestNewInterceptedPeerAuthenticationDataFactory(t *testing.T) { assert.Nil(t, ipadf) assert.Equal(t, process.ErrNilMarshalizer, err) }) + t.Run("nil CryptoComponents should error", func(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createMockComponentHolders() + arg := createMockArgument(coreComp, cryptoComp) + arg.CryptoComponents = nil + + ipadf, err := NewInterceptedPeerAuthenticationDataFactory(*arg) + assert.Nil(t, ipadf) + assert.Equal(t, process.ErrNilCryptoComponentsHolder, err) + }) t.Run("nil NodesCoordinator should error", func(t *testing.T) { t.Parallel() From 8e28ad5ff7203c803c402c200d0b484b3086db73 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 14:53:44 +0300 Subject: [PATCH 091/116] fix tests --- .../processor/peerAuthenticationInterceptorProcessor_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go index 1e402d7b220..263a5787998 100644 --- a/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go +++ b/process/interceptors/processor/peerAuthenticationInterceptorProcessor_test.go @@ -66,6 +66,7 @@ func createMockInterceptedPeerAuthentication() process.InterceptedData { PeerAuthCacher: cache.NewCacherStub(), PeerAuthenticationTimeBetweenSendsInSec: 10, ManagedPeersHolder: &testscommon.ManagedPeersHolderStub{}, + SelfPeerID: "self", } arg.DataBuff, _ = arg.Marshaller.Marshal(createInterceptedPeerAuthentication()) ipa, _ := heartbeat.NewInterceptedPeerAuthentication(arg) From d285c335f52072cd11eb87068b98e32de3feb67d Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 27 May 2026 17:41:22 +0300 Subject: [PATCH 092/116] updated deps to tags --- go.mod | 8 ++------ go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index f1a9313fa6c..6256916351d 100644 --- a/go.mod +++ b/go.mod @@ -22,8 +22,8 @@ require ( github.com/multiversx/mx-chain-es-indexer-go v1.9.3 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 - github.com/multiversx/mx-chain-storage-go v1.1.1-0.20260514073036-7edefb9fa687 - github.com/multiversx/mx-chain-vm-common-go v1.6.5 + github.com/multiversx/mx-chain-storage-go v1.1.1 + github.com/multiversx/mx-chain-vm-common-go v1.6.7 github.com/multiversx/mx-chain-vm-go v1.5.45 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 github.com/multiversx/mx-chain-vm-v1_3-go v1.3.70 @@ -39,8 +39,6 @@ require ( gopkg.in/go-playground/validator.v8 v8.18.2 ) -replace github.com/multiversx/mx-chain-vm-common-go v1.6.5 => github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a - require ( github.com/TwiN/go-color v1.1.0 // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect @@ -210,5 +208,3 @@ require ( ) replace github.com/gogo/protobuf => github.com/multiversx/protobuf v1.3.2 - -replace github.com/multiversx/mx-chain-storage-go => github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a diff --git a/go.sum b/go.sum index c9d06a55d01..c0393302443 100644 --- a/go.sum +++ b/go.sum @@ -411,10 +411,10 @@ github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/ github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a h1:zCJjTv47zA6+1s9aNTqubfd8ntOtoiBBZDEwLOoN4Z8= -github.com/multiversx/mx-chain-storage-go-ghsa-r72p-f4p9-q3j3 v1.1.1-0.20260520110037-32f823a1dc3a/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= -github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a h1:arc/Q+8Q8F1GnCGCtJu+sWnxzBmPgqtiIdzNk6tJ1T8= -github.com/multiversx/mx-chain-vm-common-go-ghsa-7cf5-cp7g-c42h v1.6.7-0.20260515121036-1c5e258de15a/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= +github.com/multiversx/mx-chain-storage-go v1.1.1 h1:Ko29uUNSRCxqkl3l8+4aIy4BKGi+k6wv8h11SxcXWyY= +github.com/multiversx/mx-chain-storage-go v1.1.1/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-vm-common-go v1.6.7 h1:oX2/RMXdhqUkJSebK+cosknBjNBX0DFAEDR6ZqNTN80= +github.com/multiversx/mx-chain-vm-common-go v1.6.7/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= github.com/multiversx/mx-chain-vm-go v1.5.45/go.mod h1:Qc2Sckw+EfQwnapkzghFfhuUAOGv29oSZgvj8LJ+xWQ= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 h1:5gSR3IMw1mcp/v5oO+vZ5YOyWO8w7O2qKhCKNPwsWNE= From adf0114a3090dfacc2eb9c3a0cc86124f203d3c4 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 27 May 2026 17:51:56 +0300 Subject: [PATCH 093/116] fix linter in tests --- consensus/spos/export_test.go | 2 +- consensus/spos/worker_internal_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/spos/export_test.go b/consensus/spos/export_test.go index 3a7e053a2fd..3f85da50554 100644 --- a/consensus/spos/export_test.go +++ b/consensus/spos/export_test.go @@ -188,7 +188,7 @@ func (wrk *Worker) SetEnableEpochsHandler(enableEpochsHandler common.EnableEpoch // AddBlockToPool - func (wrk *Worker) AddBlockToPool(bodyBytes []byte) { - wrk.addBlockToPool(bodyBytes) + _ = wrk.addBlockToPool(bodyBytes) } // AddFutureHeaderToProcessIfNeeded - diff --git a/consensus/spos/worker_internal_test.go b/consensus/spos/worker_internal_test.go index dc3fca3c0da..b3c56fee041 100644 --- a/consensus/spos/worker_internal_test.go +++ b/consensus/spos/worker_internal_test.go @@ -44,7 +44,7 @@ func TestWorker_AddBlockToPoolSkipsNonWhitelistedCrossShardMiniBlocks(t *testing }, } - worker.addBlockToPool([]byte("body")) + _ = worker.addBlockToPool([]byte("body")) require.False(t, putCalled) } @@ -87,7 +87,7 @@ func TestWorker_AddBlockToPoolAcceptsWhitelistedCrossShardMiniBlocks(t *testing. }, } - worker.addBlockToPool([]byte("body")) + _ = worker.addBlockToPool([]byte("body")) require.True(t, putCalled) } From 1ba932c8e5da4c2e6f994052a5adc63375cae212 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Thu, 28 May 2026 14:45:40 +0300 Subject: [PATCH 094/116] recreate interceptors on bootstrap --- epochStart/bootstrap/export_test.go | 23 +++ epochStart/bootstrap/process.go | 108 ++++++++++++-- epochStart/bootstrap/process_test.go | 205 +++++++++++++++++++++++++++ 3 files changed, 327 insertions(+), 9 deletions(-) diff --git a/epochStart/bootstrap/export_test.go b/epochStart/bootstrap/export_test.go index da76995a804..020c838d8cc 100644 --- a/epochStart/bootstrap/export_test.go +++ b/epochStart/bootstrap/export_test.go @@ -2,6 +2,9 @@ package bootstrap import ( "github.com/multiversx/mx-chain-core-go/data" + + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/process" ) func (e *epochStartMetaSyncer) SetEpochStartMetaBlockInterceptorProcessor(proc EpochStartMetaBlockInterceptorProcessor) { @@ -14,3 +17,23 @@ func (e *epochStartMetaBlockProcessor) GetMapMetaBlock() map[string]data.MetaHea return e.mapReceivedMetaBlocks } + +func (e *epochStartBootstrap) RebuildNetworkComponentsForShard() error { + return e.rebuildNetworkComponentsForShard() +} + +func (e *epochStartBootstrap) ResolversContainer() dataRetriever.ResolversContainer { + return e.resolversContainer +} + +func (e *epochStartBootstrap) MainInterceptorContainer() process.InterceptorsContainer { + return e.mainInterceptorContainer +} + +func (e *epochStartBootstrap) FullArchiveInterceptorContainer() process.InterceptorsContainer { + return e.fullArchiveInterceptorContainer +} + +func (e *epochStartBootstrap) RequestHandler() process.RequestHandler { + return e.requestHandler +} diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index b4f2dd14cba..e5fe97532d6 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -130,6 +130,7 @@ type epochStartBootstrap struct { requestHandler process.RequestHandler mainInterceptorContainer process.InterceptorsContainer fullArchiveInterceptorContainer process.InterceptorsContainer + resolversContainer dataRetriever.ResolversContainer dataPool dataRetriever.PoolsHolder miniBlocksSyncer epochStart.PendingMiniBlocksSyncHandler headersSyncer epochStart.HeadersByHashSyncer @@ -407,14 +408,18 @@ func (e *epochStartBootstrap) Bootstrap() (Parameters, error) { } defer func() { - errClose := e.mainInterceptorContainer.Close() - if errClose != nil { - log.Warn("prepareEpochFromStorage mainInterceptorContainer.Close()", "error", errClose) + if !check.IfNil(e.mainInterceptorContainer) { + errClose := e.mainInterceptorContainer.Close() + if errClose != nil { + log.Warn("prepareEpochFromStorage mainInterceptorContainer.Close()", "error", errClose) + } } - errClose = e.fullArchiveInterceptorContainer.Close() - if errClose != nil { - log.Warn("prepareEpochFromStorage fullArchiveInterceptorContainer.Close()", "error", errClose) + if !check.IfNil(e.fullArchiveInterceptorContainer) { + errClose := e.fullArchiveInterceptorContainer.Close() + if errClose != nil { + log.Warn("prepareEpochFromStorage fullArchiveInterceptorContainer.Close()", "error", errClose) + } } }() @@ -675,6 +680,86 @@ func (e *epochStartBootstrap) createSyncers() error { return nil } +// rebuildNetworkComponentsForShard must be called after e.shardCoordinator is reassigned to the +// node's discovered destination shard ID. +func (e *epochStartBootstrap) rebuildNetworkComponentsForShard() error { + // Nothing to rebuild when the bootstrap network stack was never set up (unit fixtures that + // invoke requestAndProcessing in isolation). In production both fields are non-nil here. + if check.IfNil(e.mainInterceptorContainer) && check.IfNil(e.resolversContainer) { + return nil + } + + log.Debug("rebuilding bootstrap network components for resolved shard", "shard", e.shardCoordinator.SelfId()) + + e.tearDownStaleNetworkComponents() + + err := e.createResolversContainer() + if err != nil { + return err + } + + err = e.createRequestHandler() + if err != nil { + return err + } + + return e.createSyncers() +} + +func (e *epochStartBootstrap) tearDownStaleNetworkComponents() { + e.unregisterInterceptorTopics(e.mainInterceptorContainer) + e.unregisterInterceptorTopics(e.fullArchiveInterceptorContainer) + e.unregisterResolverTopics(e.resolversContainer) + + if !check.IfNil(e.mainInterceptorContainer) { + errClose := e.mainInterceptorContainer.Close() + if errClose != nil { + log.Warn("rebuildNetworkComponentsForShard mainInterceptorContainer.Close()", "error", errClose) + } + e.mainInterceptorContainer = nil + } + + if !check.IfNil(e.fullArchiveInterceptorContainer) { + errClose := e.fullArchiveInterceptorContainer.Close() + if errClose != nil { + log.Warn("rebuildNetworkComponentsForShard fullArchiveInterceptorContainer.Close()", "error", errClose) + } + e.fullArchiveInterceptorContainer = nil + } + + if !check.IfNil(e.resolversContainer) { + errClose := e.resolversContainer.Close() + if errClose != nil { + log.Warn("rebuildNetworkComponentsForShard resolversContainer.Close()", "error", errClose) + } + e.resolversContainer = nil + } +} + +func (e *epochStartBootstrap) unregisterInterceptorTopics(container process.InterceptorsContainer) { + if check.IfNil(container) { + return + } + + container.Iterate(func(key string, _ process.Interceptor) bool { + log.LogIfError(e.mainMessenger.UnregisterMessageProcessor(key, common.DefaultInterceptorsIdentifier)) + log.LogIfError(e.fullArchiveMessenger.UnregisterMessageProcessor(key, common.DefaultInterceptorsIdentifier)) + return true + }) +} + +func (e *epochStartBootstrap) unregisterResolverTopics(container dataRetriever.ResolversContainer) { + if check.IfNil(container) { + return + } + + container.Iterate(func(key string, _ dataRetriever.Resolver) bool { + log.LogIfError(e.mainMessenger.UnregisterMessageProcessor(key, common.DefaultResolversIdentifier)) + log.LogIfError(e.fullArchiveMessenger.UnregisterMessageProcessor(key, common.DefaultResolversIdentifier)) + return true + }) +} + func (e *epochStartBootstrap) syncHeadersFrom(meta data.MetaHeaderHandler) (map[string]data.HeaderHandler, error) { hashesToRequest := make([][]byte, 0, len(meta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers())+1) shardIds := make([]uint32, 0, len(meta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers())+1) @@ -895,6 +980,11 @@ func (e *epochStartBootstrap) requestAndProcessing() (Parameters, error) { } log.Debug("start in epoch bootstrap: shardCoordinator", "numOfShards", e.baseData.numberOfShards, "shardId", e.baseData.shardId) + err = e.rebuildNetworkComponentsForShard() + if err != nil { + return Parameters{}, err + } + consensusTopic := common.ConsensusTopic + e.shardCoordinator.CommunicationIdentifier(e.shardCoordinator.SelfId()) err = e.mainMessenger.CreateTopic(consensusTopic, true) if err != nil { @@ -1472,9 +1562,7 @@ func (e *epochStartBootstrap) createResolversContainer() error { return err } - // TODO - create a dedicated request handler to be used when fetching required data with the correct shard coordinator - // this one should only be used before determining the correct shard where the node should reside - log.Debug("epochStartBootstrap.createRequestHandler", "shard", e.shardCoordinator.SelfId()) + log.Debug("epochStartBootstrap.createResolversContainer", "shard", e.shardCoordinator.SelfId()) resolversContainerArgs := resolverscontainer.FactoryArgs{ ShardCoordinator: e.shardCoordinator, MainMessenger: e.mainMessenger, @@ -1509,6 +1597,8 @@ func (e *epochStartBootstrap) createResolversContainer() error { return err } + e.resolversContainer = container + return resolverFactory.AddShardTrieNodeResolvers(container) } diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 239633fb312..8f49613b645 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -32,6 +32,7 @@ import ( "github.com/multiversx/mx-chain-go/epochStart/bootstrap/disabled" "github.com/multiversx/mx-chain-go/epochStart/bootstrap/types" "github.com/multiversx/mx-chain-go/epochStart/mock" + "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" processMock "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/sharding" @@ -58,6 +59,7 @@ import ( statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" storageMocks "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/syncer" + trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" validatorInfoCacherStub "github.com/multiversx/mx-chain-go/testscommon/validatorInfoCacher" "github.com/multiversx/mx-chain-go/trie/factory" updateMock "github.com/multiversx/mx-chain-go/update/mock" @@ -1022,6 +1024,209 @@ func TestCreateSyncers(t *testing.T) { assert.Nil(t, err) } +func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_NoopWhenNotInitialized(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createComponentsForEpochStart() + args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) + + epochStartProvider, _ := NewEpochStartBootstrap(args) + epochStartProvider.shardCoordinator = mock.NewMultipleShardsCoordinatorMock() + + err := epochStartProvider.RebuildNetworkComponentsForShard() + assert.Nil(t, err) + assert.True(t, check.IfNil(epochStartProvider.MainInterceptorContainer())) + assert.True(t, check.IfNil(epochStartProvider.FullArchiveInterceptorContainer())) + assert.True(t, check.IfNil(epochStartProvider.ResolversContainer())) +} + +func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordinator(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createComponentsForEpochStart() + args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) + + registeredInterceptors := make(map[string]struct{}) + unregisteredInterceptors := make(map[string]struct{}) + registeredResolvers := make(map[string]struct{}) + unregisteredResolvers := make(map[string]struct{}) + + args.MainMessenger = &p2pmocks.MessengerStub{ + RegisterMessageProcessorCalled: func(topic string, identifier string, _ p2p.MessageProcessor) error { + switch identifier { + case common.DefaultInterceptorsIdentifier: + registeredInterceptors[topic] = struct{}{} + case common.DefaultResolversIdentifier: + registeredResolvers[topic] = struct{}{} + } + return nil + }, + UnregisterMessageProcessorCalled: func(topic string, identifier string) error { + switch identifier { + case common.DefaultInterceptorsIdentifier: + unregisteredInterceptors[topic] = struct{}{} + case common.DefaultResolversIdentifier: + unregisteredResolvers[topic] = struct{}{} + } + return nil + }, + ConnectedPeersCalled: func() []core.PeerID { + return []core.PeerID{"peer0", "peer1", "peer2"} + }, + } + args.FullArchiveMessenger = &p2pmocks.MessengerStub{} + + epochStartProvider, _ := NewEpochStartBootstrap(args) + + // Shard-to-shard rather than Meta-to-shard: the rebuild's mechanics are identical, but a Meta + // initial coordinator would require populated trie roots that aren't relevant to this test. + staleCoordinator, errCoord := sharding.NewMultiShardCoordinator(2, 0) + require.Nil(t, errCoord) + epochStartProvider.shardCoordinator = staleCoordinator + epochStartProvider.dataPool = buildRebuildTestDataPool() + epochStartProvider.whiteListHandler = &testscommon.WhiteListHandlerStub{} + epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} + epochStartProvider.storageService = &storageMocks.ChainStorerStub{} + epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{} + epochStartProvider.trieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + + require.Nil(t, epochStartProvider.createResolversContainer()) + require.Nil(t, epochStartProvider.createRequestHandler()) + require.Nil(t, epochStartProvider.createSyncers()) + + require.False(t, check.IfNil(epochStartProvider.MainInterceptorContainer())) + require.False(t, check.IfNil(epochStartProvider.ResolversContainer())) + + oldInterceptorTopics := collectInterceptorTopics(epochStartProvider.MainInterceptorContainer()) + oldResolverTopics := collectResolverTopics(epochStartProvider.ResolversContainer()) + require.NotEmpty(t, oldInterceptorTopics) + require.NotEmpty(t, oldResolverTopics) + + oldMainInterceptor := epochStartProvider.MainInterceptorContainer() + oldResolvers := epochStartProvider.ResolversContainer() + oldRequestHandler := epochStartProvider.RequestHandler() + + // Reset trackers so only registrations made during the rebuild are observed + registeredInterceptors = make(map[string]struct{}) + registeredResolvers = make(map[string]struct{}) + + newCoordinator, errCoord := sharding.NewMultiShardCoordinator(2, 1) + require.Nil(t, errCoord) + epochStartProvider.shardCoordinator = newCoordinator + + err := epochStartProvider.RebuildNetworkComponentsForShard() + require.Nil(t, err) + + assert.NotSame(t, oldMainInterceptor, epochStartProvider.MainInterceptorContainer()) + assert.NotSame(t, oldResolvers, epochStartProvider.ResolversContainer()) + assert.NotSame(t, oldRequestHandler, epochStartProvider.RequestHandler()) + + for topic := range oldInterceptorTopics { + _, ok := unregisteredInterceptors[topic] + assert.True(t, ok, "interceptor topic %q should have been unregistered", topic) + } + for topic := range oldResolverTopics { + _, ok := unregisteredResolvers[topic] + assert.True(t, ok, "resolver topic %q should have been unregistered", topic) + } + + assert.NotEmpty(t, registeredInterceptors) + assert.NotEmpty(t, registeredResolvers) +} + +func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_ErrorPropagatesAndLeavesNoHalfState(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createComponentsForEpochStart() + args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) + + epochStartProvider, _ := NewEpochStartBootstrap(args) + + staleCoordinator, errCoord := sharding.NewMultiShardCoordinator(2, 0) + require.Nil(t, errCoord) + epochStartProvider.shardCoordinator = staleCoordinator + epochStartProvider.dataPool = buildRebuildTestDataPool() + epochStartProvider.whiteListHandler = &testscommon.WhiteListHandlerStub{} + epochStartProvider.whiteListerVerifiedTxs = &testscommon.WhiteListHandlerStub{} + epochStartProvider.storageService = &storageMocks.ChainStorerStub{} + epochStartProvider.interceptedDataVerifierFactory = &processMock.InterceptedDataVerifierFactoryMock{} + epochStartProvider.trieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + + require.Nil(t, epochStartProvider.createResolversContainer()) + require.Nil(t, epochStartProvider.createRequestHandler()) + require.Nil(t, epochStartProvider.createSyncers()) + + // Inject a failure into the createSyncers step of the rebuild + epochStartProvider.interceptedDataVerifierFactory = nil + + newCoordinator, errCoord := sharding.NewMultiShardCoordinator(2, 1) + require.Nil(t, errCoord) + epochStartProvider.shardCoordinator = newCoordinator + + err := epochStartProvider.RebuildNetworkComponentsForShard() + require.NotNil(t, err) + + // Tear-down ran before the failure point, so deferred Bootstrap cleanup will not double-close + assert.True(t, check.IfNil(epochStartProvider.MainInterceptorContainer())) + assert.True(t, check.IfNil(epochStartProvider.FullArchiveInterceptorContainer())) +} + +func buildRebuildTestDataPool() dataRetriever.PoolsHolder { + return &dataRetrieverMock.PoolsHolderStub{ + HeadersCalled: func() dataRetriever.HeadersPool { + return &mock.HeadersCacherStub{} + }, + TransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return testscommon.NewShardedDataStub() + }, + UnsignedTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return testscommon.NewShardedDataStub() + }, + RewardTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { + return testscommon.NewShardedDataStub() + }, + MiniBlocksCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, + TrieNodesCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, + PeerAuthenticationsCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, + HeartbeatsCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, + ProofsCalled: func() dataRetriever.ProofsPool { + return &dataRetrieverMock.ProofsPoolMock{} + }, + } +} + +func collectInterceptorTopics(container process.InterceptorsContainer) map[string]struct{} { + topics := make(map[string]struct{}) + if check.IfNil(container) { + return topics + } + container.Iterate(func(key string, _ process.Interceptor) bool { + topics[key] = struct{}{} + return true + }) + return topics +} + +func collectResolverTopics(container dataRetriever.ResolversContainer) map[string]struct{} { + topics := make(map[string]struct{}) + if check.IfNil(container) { + return topics + } + container.Iterate(func(key string, _ dataRetriever.Resolver) bool { + topics[key] = struct{}{} + return true + }) + return topics +} + func TestSyncHeadersFrom_MockHeadersSyncerShouldSyncHeaders(t *testing.T) { hdrHash1 := []byte("hdrHash1") hdrHash2 := []byte("hdrHash2") From 05f1e3211d01f764bf675e74df73148e6648c9d0 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Thu, 28 May 2026 15:15:21 +0300 Subject: [PATCH 095/116] fixes after review --- epochStart/bootstrap/process.go | 6 ++++-- epochStart/bootstrap/process_test.go | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index e5fe97532d6..d7b723d143e 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -753,9 +753,11 @@ func (e *epochStartBootstrap) unregisterResolverTopics(container dataRetriever.R return } + // Container keys are base topics; resolvers register their processor on key+_REQUEST. container.Iterate(func(key string, _ dataRetriever.Resolver) bool { - log.LogIfError(e.mainMessenger.UnregisterMessageProcessor(key, common.DefaultResolversIdentifier)) - log.LogIfError(e.fullArchiveMessenger.UnregisterMessageProcessor(key, common.DefaultResolversIdentifier)) + requestTopic := key + core.TopicRequestSuffix + log.LogIfError(e.mainMessenger.UnregisterMessageProcessor(requestTopic, common.DefaultResolversIdentifier)) + log.LogIfError(e.fullArchiveMessenger.UnregisterMessageProcessor(requestTopic, common.DefaultResolversIdentifier)) return true }) } diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 8f49613b645..b6868f0610a 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -1051,12 +1051,20 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi registeredResolvers := make(map[string]struct{}) unregisteredResolvers := make(map[string]struct{}) + // Mimic the libp2p messenger: reject duplicate (topic, identifier) registrations so that a + // missed unregister during the rebuild is caught as test failure. args.MainMessenger = &p2pmocks.MessengerStub{ RegisterMessageProcessorCalled: func(topic string, identifier string, _ p2p.MessageProcessor) error { switch identifier { case common.DefaultInterceptorsIdentifier: + if _, dup := registeredInterceptors[topic]; dup { + return fmt.Errorf("topic %q already has an interceptor processor", topic) + } registeredInterceptors[topic] = struct{}{} case common.DefaultResolversIdentifier: + if _, dup := registeredResolvers[topic]; dup { + return fmt.Errorf("topic %q already has a resolver processor", topic) + } registeredResolvers[topic] = struct{}{} } return nil @@ -1064,8 +1072,10 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi UnregisterMessageProcessorCalled: func(topic string, identifier string) error { switch identifier { case common.DefaultInterceptorsIdentifier: + delete(registeredInterceptors, topic) unregisteredInterceptors[topic] = struct{}{} case common.DefaultResolversIdentifier: + delete(registeredResolvers, topic) unregisteredResolvers[topic] = struct{}{} } return nil @@ -1106,10 +1116,6 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi oldResolvers := epochStartProvider.ResolversContainer() oldRequestHandler := epochStartProvider.RequestHandler() - // Reset trackers so only registrations made during the rebuild are observed - registeredInterceptors = make(map[string]struct{}) - registeredResolvers = make(map[string]struct{}) - newCoordinator, errCoord := sharding.NewMultiShardCoordinator(2, 1) require.Nil(t, errCoord) epochStartProvider.shardCoordinator = newCoordinator @@ -1126,8 +1132,8 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi assert.True(t, ok, "interceptor topic %q should have been unregistered", topic) } for topic := range oldResolverTopics { - _, ok := unregisteredResolvers[topic] - assert.True(t, ok, "resolver topic %q should have been unregistered", topic) + _, ok := unregisteredResolvers[topic+core.TopicRequestSuffix] + assert.True(t, ok, "resolver request topic %q should have been unregistered", topic+core.TopicRequestSuffix) } assert.NotEmpty(t, registeredInterceptors) From 96b9d52f2fbfa6e1442e3156547fd80369675540 Mon Sep 17 00:00:00 2001 From: radu Date: Thu, 28 May 2026 16:52:12 +0300 Subject: [PATCH 096/116] more bootstrap optimizations --- epochStart/bootstrap/process.go | 11 +++- epochStart/bootstrap/process_test.go | 18 ++++++ epochStart/bootstrap/storageProcess.go | 72 +++++++++++++++++---- epochStart/bootstrap/storageProcess_test.go | 34 ++++++++++ 4 files changed, 118 insertions(+), 17 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index d7b723d143e..6b9bb099b20 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -702,6 +702,7 @@ func (e *epochStartBootstrap) rebuildNetworkComponentsForShard() error { if err != nil { return err } + e.requestHandler.SetEpoch(e.epochStartMeta.GetEpoch()) return e.createSyncers() } @@ -976,15 +977,19 @@ func (e *epochStartBootstrap) requestAndProcessing() (Parameters, error) { log.Debug("start in epoch bootstrap: processNodesConfig") e.saveSelfShardId() + oldShardID := e.shardCoordinator.SelfId() + oldNumberOfShards := e.shardCoordinator.NumberOfShards() e.shardCoordinator, err = sharding.NewMultiShardCoordinator(e.baseData.numberOfShards, e.baseData.shardId) if err != nil { return Parameters{}, fmt.Errorf("%w numberOfShards=%v shardId=%v", err, e.baseData.numberOfShards, e.baseData.shardId) } log.Debug("start in epoch bootstrap: shardCoordinator", "numOfShards", e.baseData.numberOfShards, "shardId", e.baseData.shardId) - err = e.rebuildNetworkComponentsForShard() - if err != nil { - return Parameters{}, err + if oldShardID != e.shardCoordinator.SelfId() || oldNumberOfShards != e.shardCoordinator.NumberOfShards() { + err = e.rebuildNetworkComponentsForShard() + if err != nil { + return Parameters{}, err + } } consensusTopic := common.ConsensusTopic + e.shardCoordinator.CommunicationIdentifier(e.shardCoordinator.SelfId()) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index b6868f0610a..bdf43ac2dfa 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -1050,6 +1050,8 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi unregisteredInterceptors := make(map[string]struct{}) registeredResolvers := make(map[string]struct{}) unregisteredResolvers := make(map[string]struct{}) + expectedEpoch := uint32(37) + requestedEpoch := uint32(0) // Mimic the libp2p messenger: reject duplicate (topic, identifier) registrations so that a // missed unregister during the rebuild is caught as test failure. @@ -1083,10 +1085,22 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi ConnectedPeersCalled: func() []core.PeerID { return []core.PeerID{"peer0", "peer1", "peer2"} }, + ConnectedPeersOnTopicCalled: func(_ string) []core.PeerID { + return []core.PeerID{"peer0"} + }, + SendToConnectedPeerCalled: func(_ string, buff []byte, _ core.PeerID) error { + requestData := &dataRetriever.RequestData{} + err := coreComp.InternalMarshalizer().Unmarshal(requestData, buff) + assert.Nil(t, err) + requestedEpoch = requestData.Epoch + + return nil + }, } args.FullArchiveMessenger = &p2pmocks.MessengerStub{} epochStartProvider, _ := NewEpochStartBootstrap(args) + epochStartProvider.epochStartMeta = &block.MetaBlock{Epoch: expectedEpoch} // Shard-to-shard rather than Meta-to-shard: the rebuild's mechanics are identical, but a Meta // initial coordinator would require populated trie roots that aren't relevant to this test. @@ -1138,6 +1152,9 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi assert.NotEmpty(t, registeredInterceptors) assert.NotEmpty(t, registeredResolvers) + + epochStartProvider.RequestHandler().RequestMiniBlock(0, []byte("hash")) + assert.Equal(t, expectedEpoch, requestedEpoch) } func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_ErrorPropagatesAndLeavesNoHalfState(t *testing.T) { @@ -1147,6 +1164,7 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_ErrorPropagatesAnd args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) epochStartProvider, _ := NewEpochStartBootstrap(args) + epochStartProvider.epochStartMeta = &block.MetaBlock{Epoch: 37} staleCoordinator, errCoord := sharding.NewMultiShardCoordinator(2, 0) require.Nil(t, errCoord) diff --git a/epochStart/bootstrap/storageProcess.go b/epochStart/bootstrap/storageProcess.go index 90b9b98ffbd..b3ed66c3930 100644 --- a/epochStart/bootstrap/storageProcess.go +++ b/epochStart/bootstrap/storageProcess.go @@ -83,18 +83,9 @@ func (sesb *storageEpochStartBootstrap) Bootstrap() (Parameters, error) { defer func() { sesb.cleanupOnBootstrapFinish() - if !check.IfNil(sesb.container) { - err := sesb.container.Close() - if err != nil { - log.Debug("non critical error closing requesters", "error", err) - } - } - - if !check.IfNil(sesb.store) { - err := sesb.store.CloseAll() - if err != nil { - log.Debug("non critical error closing storage service", "error", err) - } + err := sesb.closeStorageRequesters() + if err != nil { + log.Debug("non critical error closing storage requesters", "error", err) } }() @@ -222,7 +213,7 @@ func (sesb *storageEpochStartBootstrap) createStorageRequestHandler() error { requestedItemsHandler, sesb.whiteListHandler, maxToRequest, - core.MetachainShardId, + sesb.shardCoordinator.SelfId(), timeBetweenRequests, time.Duration(sesb.generalConfig.Requesters.RequestProofByNonceDelayMs)*time.Millisecond, ) @@ -235,7 +226,7 @@ func (sesb *storageEpochStartBootstrap) createStorageRequesters() error { return err } - shardCoordinator, err := sharding.NewMultiShardCoordinator(sesb.genesisShardCoordinator.NumberOfShards(), sesb.genesisShardCoordinator.SelfId()) + shardCoordinator, err := sharding.NewMultiShardCoordinator(sesb.shardCoordinator.NumberOfShards(), sesb.shardCoordinator.SelfId()) if err != nil { return err } @@ -302,6 +293,50 @@ func (sesb *storageEpochStartBootstrap) createStoreForStorageResolvers(shardCoor ) } +func (sesb *storageEpochStartBootstrap) rebuildStorageComponentsForShard() error { + // Nothing to rebuild when the bootstrap stack was never set up by the storage bootstrap flow. + if check.IfNil(sesb.mainInterceptorContainer) && check.IfNil(sesb.container) { + return nil + } + + log.Debug("rebuilding storage bootstrap components for resolved shard", "shard", sesb.shardCoordinator.SelfId()) + + sesb.tearDownStaleNetworkComponents() + err := sesb.closeStorageRequesters() + if err != nil { + return err + } + + err = sesb.createStorageRequestHandler() + if err != nil { + return err + } + sesb.requestHandler.SetEpoch(sesb.epochStartMeta.GetEpoch()) + + return sesb.createSyncers() +} + +func (sesb *storageEpochStartBootstrap) closeStorageRequesters() error { + var errFound error + if !check.IfNil(sesb.container) { + err := sesb.container.Close() + if err != nil { + errFound = fmt.Errorf("close storage requesters container: %w", err) + } + sesb.container = nil + } + + if !check.IfNil(sesb.store) { + err := sesb.store.CloseAll() + if err != nil { + errFound = fmt.Errorf("close storage service: %w", err) + } + sesb.store = nil + } + + return errFound +} + func (sesb *storageEpochStartBootstrap) requestAndProcessFromStorage() (Parameters, error) { var err error sesb.baseData.numberOfShards = uint32(len(sesb.epochStartMeta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers())) @@ -332,12 +367,21 @@ func (sesb *storageEpochStartBootstrap) requestAndProcessFromStorage() (Paramete log.Debug("start in epoch bootstrap: processNodesConfig") sesb.saveSelfShardId() + oldShardID := sesb.shardCoordinator.SelfId() + oldNumberOfShards := sesb.shardCoordinator.NumberOfShards() sesb.shardCoordinator, err = sharding.NewMultiShardCoordinator(sesb.baseData.numberOfShards, sesb.baseData.shardId) if err != nil { return Parameters{}, fmt.Errorf("%w numberOfShards=%v shardId=%v", err, sesb.baseData.numberOfShards, sesb.baseData.shardId) } log.Debug("start in epoch bootstrap: shardCoordinator", "numOfShards", sesb.baseData.numberOfShards, "shardId", sesb.baseData.shardId) + if oldShardID != sesb.shardCoordinator.SelfId() || oldNumberOfShards != sesb.shardCoordinator.NumberOfShards() { + err = sesb.rebuildStorageComponentsForShard() + if err != nil { + return Parameters{}, err + } + } + consensusTopic := common.ConsensusTopic + sesb.shardCoordinator.CommunicationIdentifier(sesb.shardCoordinator.SelfId()) err = sesb.mainMessenger.CreateTopic(consensusTopic, true) if err != nil { diff --git a/epochStart/bootstrap/storageProcess_test.go b/epochStart/bootstrap/storageProcess_test.go index 34a7f97cbcc..66e0a86c5f1 100644 --- a/epochStart/bootstrap/storageProcess_test.go +++ b/epochStart/bootstrap/storageProcess_test.go @@ -16,7 +16,9 @@ import ( "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/mock" "github.com/multiversx/mx-chain-go/process" + processFactory "github.com/multiversx/mx-chain-go/process/factory" processMock "github.com/multiversx/mx-chain-go/process/mock" + "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" @@ -68,6 +70,38 @@ func TestNewStorageEpochStartBootstrap_ShouldWork(t *testing.T) { assert.Nil(t, err) } +func TestStorageEpochStartBootstrap_CreateStorageRequestHandlerUsesCurrentShard(t *testing.T) { + t.Parallel() + + coreComp, cryptoComp := createComponentsForEpochStart() + args := createMockStorageEpochStartBootstrapArgs(coreComp, cryptoComp) + args.GeneralConfig = testscommon.GetGeneralConfig() + args.ImportDbConfig = config.ImportDbConfig{ + ImportDBWorkingDir: t.TempDir(), + ImportDBTargetShardID: 1, + } + + sesb, err := NewStorageEpochStartBootstrap(args) + assert.Nil(t, err) + sesb.shardCoordinator, err = sharding.NewMultiShardCoordinator(2, 1) + assert.Nil(t, err) + + err = sesb.createStorageRequestHandler() + assert.Nil(t, err) + defer func() { + _ = sesb.closeStorageRequesters() + }() + + expectedTopic := processFactory.MiniBlocksTopic + core.CommunicationIdentifierBetweenShards(0, 1) + oldTopic := processFactory.MiniBlocksTopic + core.CommunicationIdentifierBetweenShards(0, core.MetachainShardId) + + _, err = sesb.container.Get(expectedTopic) + assert.Nil(t, err) + + _, err = sesb.container.Get(oldTopic) + assert.NotNil(t, err) +} + func TestStorageEpochStartBootstrap_BootstrapStartInEpochNotEnabled(t *testing.T) { coreComp, cryptoComp := createComponentsForEpochStart() args := createMockStorageEpochStartBootstrapArgs(coreComp, cryptoComp) From 0436b9de4cbc0fd04aab9f07e97d459b7df43371 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Thu, 28 May 2026 17:39:13 +0300 Subject: [PATCH 097/116] add nil checks --- epochStart/bootstrap/process.go | 5 ++++- epochStart/bootstrap/storageProcess.go | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 6b9bb099b20..589171865db 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -702,7 +702,10 @@ func (e *epochStartBootstrap) rebuildNetworkComponentsForShard() error { if err != nil { return err } - e.requestHandler.SetEpoch(e.epochStartMeta.GetEpoch()) + + if !check.IfNil(e.epochStartMeta) { + e.requestHandler.SetEpoch(e.epochStartMeta.GetEpoch()) + } return e.createSyncers() } diff --git a/epochStart/bootstrap/storageProcess.go b/epochStart/bootstrap/storageProcess.go index b3ed66c3930..34dbfecbe73 100644 --- a/epochStart/bootstrap/storageProcess.go +++ b/epochStart/bootstrap/storageProcess.go @@ -11,7 +11,6 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" - "github.com/multiversx/mx-chain-go/process/interceptors/processor" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" @@ -23,6 +22,7 @@ import ( "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/epochStart/bootstrap/disabled" "github.com/multiversx/mx-chain-go/epochStart/notifier" + "github.com/multiversx/mx-chain-go/process/interceptors/processor" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/storage/cache" storageFactory "github.com/multiversx/mx-chain-go/storage/factory" @@ -311,8 +311,9 @@ func (sesb *storageEpochStartBootstrap) rebuildStorageComponentsForShard() error if err != nil { return err } - sesb.requestHandler.SetEpoch(sesb.epochStartMeta.GetEpoch()) - + if !check.IfNil(sesb.epochStartMeta) { + sesb.requestHandler.SetEpoch(sesb.epochStartMeta.GetEpoch()) + } return sesb.createSyncers() } From 94ed12d4dbdef7cfd84782bd869bc4bca114ee54 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Fri, 29 May 2026 15:00:26 +0300 Subject: [PATCH 098/116] unregister all processors on tear down --- epochStart/bootstrap/process.go | 33 ++++------------------------ epochStart/bootstrap/process_test.go | 26 ++++++---------------- 2 files changed, 11 insertions(+), 48 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 589171865db..13469fe333b 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -711,9 +711,10 @@ func (e *epochStartBootstrap) rebuildNetworkComponentsForShard() error { } func (e *epochStartBootstrap) tearDownStaleNetworkComponents() { - e.unregisterInterceptorTopics(e.mainInterceptorContainer) - e.unregisterInterceptorTopics(e.fullArchiveInterceptorContainer) - e.unregisterResolverTopics(e.resolversContainer) + log.LogIfError(e.mainMessenger.UnregisterAllMessageProcessors()) + log.LogIfError(e.mainMessenger.UnJoinAllTopics()) + log.LogIfError(e.fullArchiveMessenger.UnregisterAllMessageProcessors()) + log.LogIfError(e.fullArchiveMessenger.UnJoinAllTopics()) if !check.IfNil(e.mainInterceptorContainer) { errClose := e.mainInterceptorContainer.Close() @@ -740,32 +741,6 @@ func (e *epochStartBootstrap) tearDownStaleNetworkComponents() { } } -func (e *epochStartBootstrap) unregisterInterceptorTopics(container process.InterceptorsContainer) { - if check.IfNil(container) { - return - } - - container.Iterate(func(key string, _ process.Interceptor) bool { - log.LogIfError(e.mainMessenger.UnregisterMessageProcessor(key, common.DefaultInterceptorsIdentifier)) - log.LogIfError(e.fullArchiveMessenger.UnregisterMessageProcessor(key, common.DefaultInterceptorsIdentifier)) - return true - }) -} - -func (e *epochStartBootstrap) unregisterResolverTopics(container dataRetriever.ResolversContainer) { - if check.IfNil(container) { - return - } - - // Container keys are base topics; resolvers register their processor on key+_REQUEST. - container.Iterate(func(key string, _ dataRetriever.Resolver) bool { - requestTopic := key + core.TopicRequestSuffix - log.LogIfError(e.mainMessenger.UnregisterMessageProcessor(requestTopic, common.DefaultResolversIdentifier)) - log.LogIfError(e.fullArchiveMessenger.UnregisterMessageProcessor(requestTopic, common.DefaultResolversIdentifier)) - return true - }) -} - func (e *epochStartBootstrap) syncHeadersFrom(meta data.MetaHeaderHandler) (map[string]data.HeaderHandler, error) { hashesToRequest := make([][]byte, 0, len(meta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers())+1) shardIds := make([]uint32, 0, len(meta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers())+1) diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index bdf43ac2dfa..cf1487a85a5 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -1047,9 +1047,7 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) registeredInterceptors := make(map[string]struct{}) - unregisteredInterceptors := make(map[string]struct{}) registeredResolvers := make(map[string]struct{}) - unregisteredResolvers := make(map[string]struct{}) expectedEpoch := uint32(37) requestedEpoch := uint32(0) @@ -1072,14 +1070,13 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi return nil }, UnregisterMessageProcessorCalled: func(topic string, identifier string) error { - switch identifier { - case common.DefaultInterceptorsIdentifier: - delete(registeredInterceptors, topic) - unregisteredInterceptors[topic] = struct{}{} - case common.DefaultResolversIdentifier: - delete(registeredResolvers, topic) - unregisteredResolvers[topic] = struct{}{} - } + require.Fail(t, "should have not been called") + return nil + }, + UnregisterAllMessageProcessorsCalled: func() error { + registeredInterceptors = make(map[string]struct{}) + registeredResolvers = make(map[string]struct{}) + return nil }, ConnectedPeersCalled: func() []core.PeerID { @@ -1141,15 +1138,6 @@ func TestEpochStartBootstrap_RebuildNetworkComponentsForShard_RewiresStaleCoordi assert.NotSame(t, oldResolvers, epochStartProvider.ResolversContainer()) assert.NotSame(t, oldRequestHandler, epochStartProvider.RequestHandler()) - for topic := range oldInterceptorTopics { - _, ok := unregisteredInterceptors[topic] - assert.True(t, ok, "interceptor topic %q should have been unregistered", topic) - } - for topic := range oldResolverTopics { - _, ok := unregisteredResolvers[topic+core.TopicRequestSuffix] - assert.True(t, ok, "resolver request topic %q should have been unregistered", topic+core.TopicRequestSuffix) - } - assert.NotEmpty(t, registeredInterceptors) assert.NotEmpty(t, registeredResolvers) From e3c0cada27bf556afbd9423bd2f8cf31445be509 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 11:32:31 +0300 Subject: [PATCH 099/116] update versions --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- testscommon/txcachemocks/txCacheMock.go | 2 +- txcache/crossTxCache.go | 12 +++++++++--- txcache/crossTxCache_test.go | 4 +--- txcache/disabledCache.go | 6 +++++- txcache/txCache.go | 6 +++++- 7 files changed, 36 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 2c04e0e63dc..8ca0e6c5c89 100644 --- a/go.mod +++ b/go.mod @@ -18,14 +18,14 @@ require ( github.com/libp2p/go-libp2p v0.38.2 github.com/libp2p/go-libp2p-pubsub v0.13.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/multiversx/mx-chain-communication-go v1.3.1 - github.com/multiversx/mx-chain-core-go v1.5.0 + github.com/multiversx/mx-chain-communication-go v1.3.3-0.20260608072730-982186a1ad78 + github.com/multiversx/mx-chain-core-go v1.5.1-0.20260608073155-f1f550c8a612 github.com/multiversx/mx-chain-crypto-go v1.3.1 - github.com/multiversx/mx-chain-es-indexer-go v1.10.2 + github.com/multiversx/mx-chain-es-indexer-go v1.10.3-0.20260608081825-40e586306036 github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-scenario-go v1.6.0 - github.com/multiversx/mx-chain-storage-go v1.1.0 - github.com/multiversx/mx-chain-vm-common-go v1.6.6 + github.com/multiversx/mx-chain-storage-go v1.1.2-0.20260608080818-1fde35395146 + github.com/multiversx/mx-chain-vm-common-go v1.6.7 github.com/multiversx/mx-chain-vm-go v1.5.45 github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 github.com/multiversx/mx-chain-vm-v1_3-go v1.3.70 diff --git a/go.sum b/go.sum index db597d35dba..28a0b77d4a0 100644 --- a/go.sum +++ b/go.sum @@ -399,22 +399,22 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUYwbO0993uPI= github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= -github.com/multiversx/mx-chain-communication-go v1.3.1 h1:rJj4FOTqacD+yaAfz61FoEtwpAYmOQFyLEHdy1YZya4= -github.com/multiversx/mx-chain-communication-go v1.3.1/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.5.0 h1:YBxTsxBGd4hy9A3plcILu+jDy4BcQaD8oyVRDC1tz8A= -github.com/multiversx/mx-chain-core-go v1.5.0/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-communication-go v1.3.3-0.20260608072730-982186a1ad78 h1:vdYSj8Jj83H5wMkQOcDKvBXf2yVLXgVmA1IthHuv3aY= +github.com/multiversx/mx-chain-communication-go v1.3.3-0.20260608072730-982186a1ad78/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= +github.com/multiversx/mx-chain-core-go v1.5.1-0.20260608073155-f1f550c8a612 h1:Hol8/gBD3d84kIuVsss+1Zx+sUIOpshU/wHC1d2DFVI= +github.com/multiversx/mx-chain-core-go v1.5.1-0.20260608073155-f1f550c8a612/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.1 h1:tCoGkfiv0wz97kuW6AZPW4RVL0Yp7PBo8NKQj9f2oh4= github.com/multiversx/mx-chain-crypto-go v1.3.1/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= -github.com/multiversx/mx-chain-es-indexer-go v1.10.2 h1:mLFRUpZ2bWeYplU1e0kb318kk1x7AV9owq5B4XRdOqE= -github.com/multiversx/mx-chain-es-indexer-go v1.10.2/go.mod h1:HtHJx2XGnFTZE2GBcWxDiBr/DIuDsmb5R38+P3Jp87c= +github.com/multiversx/mx-chain-es-indexer-go v1.10.3-0.20260608081825-40e586306036 h1:a0euQ0LrFvP3y7Uf4CJ39tXBo0tsTEzPLrvJriuO2oA= +github.com/multiversx/mx-chain-es-indexer-go v1.10.3-0.20260608081825-40e586306036/go.mod h1:LbUYOxarVj0sHG9vPMNaiGuCRmKvEDAUxY4pnBAtzX8= github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/WI0Qh112whgZNM= github.com/multiversx/mx-chain-logger-go v1.1.0/go.mod h1:K9XgiohLwOsNACETMNL0LItJMREuEvTH6NsoXWXWg7g= github.com/multiversx/mx-chain-scenario-go v1.6.0 h1:cwDFuS1pSc4YXnfiKKDTEb+QDY4fulPQaiRgIebnKxI= github.com/multiversx/mx-chain-scenario-go v1.6.0/go.mod h1:GrSYu1SnMvsIm9djUz1X13224HcvdY6Nb5KHNT3xZPA= -github.com/multiversx/mx-chain-storage-go v1.1.0 h1:M1Y9DqMrJ62s7Zw31+cyuqsnPIvlG4jLBJl5WzeZLe8= -github.com/multiversx/mx-chain-storage-go v1.1.0/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= -github.com/multiversx/mx-chain-vm-common-go v1.6.6 h1:BJSQndP8KSqcSIi47wQwQy3uBIn5rbT3213eJroVaog= -github.com/multiversx/mx-chain-vm-common-go v1.6.6/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= +github.com/multiversx/mx-chain-storage-go v1.1.2-0.20260608080818-1fde35395146 h1:ECkaR/1fkcJhw1YMi1gqeBDEmLeyM7/veNAqyLQ+LWo= +github.com/multiversx/mx-chain-storage-go v1.1.2-0.20260608080818-1fde35395146/go.mod h1:o6Jm7cjfPmcc6XpyihYWrd6sx3sgqwurrunw3ZrfyxI= +github.com/multiversx/mx-chain-vm-common-go v1.6.7 h1:oX2/RMXdhqUkJSebK+cosknBjNBX0DFAEDR6ZqNTN80= +github.com/multiversx/mx-chain-vm-common-go v1.6.7/go.mod h1:Lc7r4VDPYRDS0CVIaWAoLtf3YQn6PZEYHv4QtaOE2Z0= github.com/multiversx/mx-chain-vm-go v1.5.45 h1:0JBB/imgI8wa6muXtdGMDrW685sdsRwH/+gMPuX96OU= github.com/multiversx/mx-chain-vm-go v1.5.45/go.mod h1:Qc2Sckw+EfQwnapkzghFfhuUAOGv29oSZgvj8LJ+xWQ= github.com/multiversx/mx-chain-vm-v1_2-go v1.2.69 h1:5gSR3IMw1mcp/v5oO+vZ5YOyWO8w7O2qKhCKNPwsWNE= diff --git a/testscommon/txcachemocks/txCacheMock.go b/testscommon/txcachemocks/txCacheMock.go index 095d205d3c9..755b97a334c 100644 --- a/testscommon/txcachemocks/txCacheMock.go +++ b/testscommon/txcachemocks/txCacheMock.go @@ -1,6 +1,6 @@ package txcachemocks -import "github.com/multiversx/mx-chain-storage-go/txcache" +import "github.com/multiversx/mx-chain-go/txcache" // TxCacheMock - type TxCacheMock struct { diff --git a/txcache/crossTxCache.go b/txcache/crossTxCache.go index eef9622bfbf..b543eb65398 100644 --- a/txcache/crossTxCache.go +++ b/txcache/crossTxCache.go @@ -48,18 +48,24 @@ func NewCrossTxCache(config ConfigDestinationMe) (*CrossTxCache, error) { return &cache, nil } -// ImmunizeTxsAgainstEviction marks items as non-evictable -func (cache *CrossTxCache) ImmunizeTxsAgainstEviction(keys [][]byte) { - numNow, numFuture := cache.ImmunityCache.ImmunizeKeys(keys) +// ImmunizeTxsAgainstEviction marks items as non-evictable for the provided confirmation nonce +func (cache *CrossTxCache) ImmunizeTxsAgainstEviction(keys [][]byte, nonce uint64) { + numNow, numFuture := cache.ImmunityCache.ImmunizeKeys(keys, nonce) log.Trace("CrossTxCache.ImmunizeTxsAgainstEviction", "name", cache.config.Name, "len(keys)", len(keys), "numNow", numNow, "numFuture", numFuture, + "nonce", nonce, ) cache.Diagnose(false) } +// SetOldestImmuneNonce deactivates immunity below the provided nonce +func (cache *CrossTxCache) SetOldestImmuneNonce(nonce uint64) { + cache.ImmunityCache.SetOldestImmuneNonce(nonce) +} + // AddTx adds a transaction in the cache func (cache *CrossTxCache) AddTx(tx *WrappedTransaction) (has, added bool) { log.Trace("CrossTxCache.AddTx", "name", cache.config.Name, "txHash", tx.TxHash) diff --git a/txcache/crossTxCache_test.go b/txcache/crossTxCache_test.go index d657e5684fb..74af4db106a 100644 --- a/txcache/crossTxCache_test.go +++ b/txcache/crossTxCache_test.go @@ -51,9 +51,7 @@ func TestCrossTxCache_DoImmunizeTxsAgainstEviction(t *testing.T) { cache := newCrossTxCacheToTest(1, 8, math.MaxUint16) cache.addTestTxs("a", "b", "c", "d") - numNow, numFuture := cache.ImmunizeKeys(hashesAsBytes([]string{"a", "b", "e", "f"})) - require.Equal(t, 2, numNow) - require.Equal(t, 2, numFuture) + cache.ImmunizeTxsAgainstEviction(hashesAsBytes([]string{"a", "b", "e", "f"}), 7) require.Equal(t, 4, cache.Len()) cache.addTestTxs("e", "f", "g", "h") diff --git a/txcache/disabledCache.go b/txcache/disabledCache.go index 874cbb0d5f2..3034fc6d54a 100644 --- a/txcache/disabledCache.go +++ b/txcache/disabledCache.go @@ -111,7 +111,11 @@ func (cache *DisabledCache) UnRegisterHandler(string) { } // ImmunizeTxsAgainstEviction does nothing -func (cache *DisabledCache) ImmunizeTxsAgainstEviction(_ [][]byte) { +func (cache *DisabledCache) ImmunizeTxsAgainstEviction(_ [][]byte, _ uint64) { +} + +// SetOldestImmuneNonce does nothing +func (cache *DisabledCache) SetOldestImmuneNonce(_ uint64) { } // Diagnose does nothing diff --git a/txcache/txCache.go b/txcache/txCache.go index 73aa1ee5078..6d92d0d625c 100644 --- a/txcache/txCache.go +++ b/txcache/txCache.go @@ -401,7 +401,11 @@ func (cache *TxCache) UnRegisterHandler(string) { } // ImmunizeTxsAgainstEviction does nothing for this type of cache -func (cache *TxCache) ImmunizeTxsAgainstEviction(_ [][]byte) { +func (cache *TxCache) ImmunizeTxsAgainstEviction(_ [][]byte, _ uint64) { +} + +// SetOldestImmuneNonce does nothing for this type of cache +func (cache *TxCache) SetOldestImmuneNonce(_ uint64) { } // Close does nothing for this cacher implementation From 87143774b3f8bfe422af5238da94016e21e21c01 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 12:58:52 +0300 Subject: [PATCH 100/116] fixes after merge --- cmd/node/config/config.toml | 5 +- config/config.go | 22 ++-- .../txpool/memorytests/memory_test.go | 6 +- dataRetriever/txpool/mempoolHost_test.go | 12 ++- dataRetriever/txpool/shardedTxPool_test.go | 27 ++--- .../epochStartInterceptorsContainerFactory.go | 2 +- factory/consensus/consensusComponents.go | 2 - factory/processing/processComponents.go | 1 + .../chainSimulator/mempool/mempool_test.go | 102 +++++++++--------- .../chainSimulator/mempool/testutils_test.go | 24 ++--- integrationTests/testFullNode.go | 15 +-- .../transactionAPI/apiTransactionProcessor.go | 8 +- .../apiTransactionProcessor_test.go | 16 +-- process/block/baseProcess.go | 61 ++--------- process/block/metablockProposal.go | 2 +- process/block/shardblockProposal.go | 2 +- process/factory/interceptorscontainer/args.go | 4 +- .../metaInterceptorsContainerFactory.go | 3 +- .../shardInterceptorsContainerFactory.go | 2 +- .../processor/trieNodeChunksProcessor.go | 1 + testscommon/dataRetriever/poolFactory.go | 4 +- testscommon/dataRetriever/poolsHolderMock.go | 4 +- testscommon/generalConfig.go | 10 +- .../accountNonceAndBalanceProviderMock.go | 3 +- .../{ => mempool}/mempoolHostMock.go | 2 +- .../{ => mempool}/selectionSessionMock.go | 3 +- .../{ => mempool}/txGasHandlerMock.go | 2 +- txcache/autoClean_test.go | 22 ++-- txcache/eviction_test.go | 16 +-- txcache/selectionTracker_test.go | 68 ++++++------ txcache/selection_test.go | 54 +++++----- txcache/testutils_test.go | 5 +- txcache/transactionsHeapItem_test.go | 10 +- txcache/txCache_test.go | 24 +++-- txcache/virtualSelectionSession_test.go | 42 ++++---- txcache/virtualSessionComputer_test.go | 6 +- txcache/wrappedTransaction_test.go | 19 ++-- update/factory/fullSyncInterceptors.go | 1 + 38 files changed, 297 insertions(+), 315 deletions(-) rename testscommon/txcachemocks/{ => mempool}/accountNonceAndBalanceProviderMock.go (99%) rename testscommon/txcachemocks/{ => mempool}/mempoolHostMock.go (99%) rename testscommon/txcachemocks/{ => mempool}/selectionSessionMock.go (99%) rename testscommon/txcachemocks/{ => mempool}/txGasHandlerMock.go (99%) diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 56b226e8b29..73cc3656348 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -660,14 +660,13 @@ [Antiflood] Enabled = true + MaxAllowedTrieNodeChunks = 10 + TrieNodeChunksInactivityTimeoutInSec = 10 [[Antiflood.ConfigsByRound]] Round = 0 NumConcurrentResolverJobs = 50 NumConcurrentResolvingTrieNodesJobs = 3 - - MaxAllowedTrieNodeChunks = 10 - TrieNodeChunksInactivityTimeoutInSec = 10 [Antiflood.ConfigsByRound.FastReacting] IntervalInSeconds = 1 ReservedPercent = 20.0 diff --git a/config/config.go b/config/config.go index 70988c008d4..e379d3512ac 100644 --- a/config/config.go +++ b/config/config.go @@ -526,8 +526,10 @@ type TxAccumulatorConfig struct { // AntifloodConfig will hold all p2p antiflood parameters type AntifloodConfig struct { - Enabled bool - ConfigsByRound []AntifloodConfigByRound + Enabled bool + MaxAllowedTrieNodeChunks uint32 + TrieNodeChunksInactivityTimeoutInSec int64 + ConfigsByRound []AntifloodConfigByRound } // AntifloodConfigByRound will hold antiflood parameters by round @@ -535,15 +537,13 @@ type AntifloodConfigByRound struct { Round uint64 NumConcurrentResolverJobs int32 NumConcurrentResolvingTrieNodesJobs int32 - MaxAllowedTrieNodeChunks uint32 - TrieNodeChunksInactivityTimeoutInSec int64 - OutOfSpecs FloodPreventerConfig - FastReacting FloodPreventerConfig - SlowReacting FloodPreventerConfig - PeerMaxOutput FloodPreventerConfig - Cache CacheConfig - Topic TopicAntifloodConfig - TxAccumulator TxAccumulatorConfig + OutOfSpecs FloodPreventerConfig + FastReacting FloodPreventerConfig + SlowReacting FloodPreventerConfig + PeerMaxOutput FloodPreventerConfig + Cache CacheConfig + Topic TopicAntifloodConfig + TxAccumulator TxAccumulatorConfig } // FloodPreventerConfig will hold all flood preventer parameters diff --git a/dataRetriever/txpool/memorytests/memory_test.go b/dataRetriever/txpool/memorytests/memory_test.go index 727cdbdca72..1a899f742ab 100644 --- a/dataRetriever/txpool/memorytests/memory_test.go +++ b/dataRetriever/txpool/memorytests/memory_test.go @@ -13,11 +13,13 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/txpool" "github.com/multiversx/mx-chain-go/storage/storageunit" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/require" ) @@ -114,7 +116,7 @@ func newPool() dataRetriever.ShardedDataCacherNotifier { args := txpool.ArgShardedTxPool{ Config: cacheConfig, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 2, SelfShardID: 0, diff --git a/dataRetriever/txpool/mempoolHost_test.go b/dataRetriever/txpool/mempoolHost_test.go index a013a88fa19..18f3baf155d 100644 --- a/dataRetriever/txpool/mempoolHost_test.go +++ b/dataRetriever/txpool/mempoolHost_test.go @@ -9,9 +9,11 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/require" ) @@ -26,14 +28,14 @@ func TestNewMempoolHost(t *testing.T) { require.ErrorIs(t, err, dataRetriever.ErrNilTxGasHandler) host, err = newMempoolHost(argsMempoolHost{ - txGasHandler: txcachemocks.NewTxGasHandlerMock(), + txGasHandler: mempool.NewTxGasHandlerMock(), marshalizer: nil, }) require.Nil(t, host) require.ErrorIs(t, err, dataRetriever.ErrNilMarshalizer) host, err = newMempoolHost(argsMempoolHost{ - txGasHandler: txcachemocks.NewTxGasHandlerMock(), + txGasHandler: mempool.NewTxGasHandlerMock(), marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.NoError(t, err) @@ -44,7 +46,7 @@ func TestMempoolHost_GetTransferredValue(t *testing.T) { t.Parallel() host, err := newMempoolHost(argsMempoolHost{ - txGasHandler: txcachemocks.NewTxGasHandlerMock(), + txGasHandler: mempool.NewTxGasHandlerMock(), marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.NoError(t, err) @@ -86,7 +88,7 @@ func TestMempoolHost_GetTransferredValue(t *testing.T) { func TestBenchmarkMempoolHost_GetTransferredValue(t *testing.T) { host, err := newMempoolHost(argsMempoolHost{ - txGasHandler: txcachemocks.NewTxGasHandlerMock(), + txGasHandler: mempool.NewTxGasHandlerMock(), marshalizer: &marshal.GogoProtoMarshalizer{}, }) require.NoError(t, err) diff --git a/dataRetriever/txpool/shardedTxPool_test.go b/dataRetriever/txpool/shardedTxPool_test.go index 482275fc2c9..63a5930a6f6 100644 --- a/dataRetriever/txpool/shardedTxPool_test.go +++ b/dataRetriever/txpool/shardedTxPool_test.go @@ -13,7 +13,10 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/config" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/require" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -42,7 +45,7 @@ func Test_NewShardedTxPool_WhenBadConfig(t *testing.T) { SizeInBytesPerSender: 40960, Shards: 16, }, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 1, TxCacheBoundsConfig: config.TxCacheBoundsConfig{ @@ -126,7 +129,7 @@ func Test_NewShardedTxPool_ComputesCacheConfig(t *testing.T) { cacheConfig := storageunit.CacheConfig{SizeInBytes: 419430400, SizeInBytesPerSender: 614400, Capacity: 600000, SizePerSender: 1000, Shards: 1} args := ArgShardedTxPool{ Config: cacheConfig, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 2, TxCacheBoundsConfig: config.TxCacheBoundsConfig{ @@ -293,7 +296,7 @@ func TestCleanupSelfShardTxCache_NilMempool(t *testing.T) { txPool := poolAsInterface.(*shardedTxPool) delete(txPool.backingMap, "0") - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() cleanupLoopMaximumDuration := time.Millisecond * 100 require.NotPanics(t, func() { @@ -308,7 +311,7 @@ func Test_Parallel_CleanupSelfShardTxCache(t *testing.T) { t.Parallel() poolAsInterface, _ := newTxPoolToTest() pool := poolAsInterface.(*shardedTxPool) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 2) accountsProvider.SetNonce([]byte("bob"), 42) accountsProvider.SetNonce([]byte("carol"), 7) @@ -341,7 +344,7 @@ func Test_CleanupSelfShardTxCache(t *testing.T) { poolAsInterface, _ := newTxPoolToTest() pool := poolAsInterface.(*shardedTxPool) cache := pool.getTxCache("0").(*txcache.TxCache) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 2) accountsProvider.SetNonce([]byte("bob"), 42) accountsProvider.SetNonce([]byte("carol"), 7) @@ -526,7 +529,7 @@ func Test_routeToCacheUnions(t *testing.T) { } args := ArgShardedTxPool{ Config: cacheConfig, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 4, SelfShardID: 42, @@ -559,7 +562,7 @@ func TestShardedTxPool_getSelfShardTxCache(t *testing.T) { } args := ArgShardedTxPool{ Config: cacheConfig, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 3, SelfShardID: 2, @@ -582,7 +585,7 @@ func TestShardedTxPool_GetNumTrackedBlocks(t *testing.T) { txCache := pool.getSelfShardTxCache() numOfBlocks := 10 - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() for i := 1; i < numOfBlocks+1; i++ { err := txCache.OnProposedBlock( @@ -610,7 +613,7 @@ func TestShardedTxPool_GetNumTrackedAccounts(t *testing.T) { txCache := pool.getSelfShardTxCache() - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte("rootHash0"), nil } @@ -692,7 +695,7 @@ func TestShardedTxPool_OnProposedBlock_And_OnExecutedBlock(t *testing.T) { } args := ArgShardedTxPool{ Config: cacheConfig, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 3, SelfShardID: 0, @@ -715,7 +718,7 @@ func TestShardedTxPool_OnProposedBlock_And_OnExecutedBlock(t *testing.T) { []byte("abba"), &block.Body{}, &block.HeaderV2{}, - txcachemocks.NewAccountNonceAndBalanceProviderMock(), + mempool.NewAccountNonceAndBalanceProviderMock(), nil, ) require.Nil(t, err) @@ -760,7 +763,7 @@ func newTxPoolToTest() (dataRetriever.ShardedDataCacherNotifier, error) { } args := ArgShardedTxPool{ Config: cacheConfig, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 4, SelfShardID: 0, diff --git a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go index 7c758867917..12f870332cd 100644 --- a/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go +++ b/epochStart/bootstrap/factory/epochStartInterceptorsContainerFactory.go @@ -114,7 +114,7 @@ func NewEpochStartInterceptorsContainer(args ArgsEpochStartInterceptorContainer) HardforkTrigger: hardforkTrigger, NodeOperationMode: args.NodeOperationMode, InterceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, - Config: args.Config, + Config: args.Config, } var interceptorsContainerFactory process.InterceptorsContainerFactory diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index a78c3ebdda2..9f846994b3a 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -167,8 +167,6 @@ func (ccf *consensusComponentsFactory) Create() (*consensusComponents, error) { ccf.processComponents.InterceptorsContainer(), ccf.coreComponents.AlarmScheduler(), ccf.cryptoComponents.KeysHandler(), - ccf.dataComponents.Datapool().Proofs(), - ccf.coreComponents.EnableEpochsHandler(), ) if err != nil { return nil, err diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index 0b1d67b4b54..40959d170c9 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -16,6 +16,7 @@ import ( dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" + "github.com/multiversx/mx-chain-core-go/data/transaction" vmcommon "github.com/multiversx/mx-chain-vm-common-go" vmcommonBuiltInFunctions "github.com/multiversx/mx-chain-vm-common-go/builtInFunctions" diff --git a/integrationTests/chainSimulator/mempool/mempool_test.go b/integrationTests/chainSimulator/mempool/mempool_test.go index f6d22147c5f..eee80ec0775 100644 --- a/integrationTests/chainSimulator/mempool/mempool_test.go +++ b/integrationTests/chainSimulator/mempool/mempool_test.go @@ -17,7 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/common/holders" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" "github.com/multiversx/mx-chain-go/txcache" "github.com/multiversx/mx-chain-go/config" @@ -504,7 +504,7 @@ func TestMempoolWithChainSimulator_Eviction(t *testing.T) { func Test_Selection_ShouldNotSelectSameTransactionsWithSameSender(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -522,12 +522,12 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithSameSender(t *testing.T) }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -609,7 +609,7 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithSameSender(t *testing.T) func Test_Selection_ShouldNotSelectSameTransactionsWithDifferentSenders(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -631,12 +631,12 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithDifferentSenders(t *testi }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -753,7 +753,7 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithDifferentSenders(t *testi func Test_Selection_ShouldNotSelectSameTransactionsWithManyTransactions(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -778,12 +778,12 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithManyTransactions(t *testi }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -851,7 +851,7 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithManyTransactions(t *testi func Test_Selection_ProposeEmptyBlocks(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -876,12 +876,12 @@ func Test_Selection_ProposeEmptyBlocks(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -973,7 +973,7 @@ func Test_Selection_ProposeBlocksWithSameNonceToTriggerForkScenarios(t *testing. t.Parallel() t.Run("should work with only one proposed block being replaced", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -998,12 +998,12 @@ func Test_Selection_ProposeBlocksWithSameNonceToTriggerForkScenarios(t *testing. }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1099,7 +1099,7 @@ func Test_Selection_ProposeBlocksWithSameNonceToTriggerForkScenarios(t *testing. }) t.Run("should work with many proposed blocks being replaced", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -1128,12 +1128,12 @@ func Test_Selection_ProposeBlocksWithSameNonceToTriggerForkScenarios(t *testing. }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1275,7 +1275,7 @@ func Test_Selection_ProposeBlocksWithSameNonceToTriggerForkScenarios(t *testing. func Test_Selection_ShouldNotSelectSameTransactionsWithManyTransactionsAndExecutedBlockNotification(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -1298,12 +1298,12 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithManyTransactionsAndExecut }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1397,7 +1397,7 @@ func Test_Selection_ShouldNotSelectSameTransactionsWithManyTransactionsAndExecut func Test_Selection_ProposeEmptyBlocksAndExecutedBlockNotification(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -1417,12 +1417,12 @@ func Test_Selection_ProposeEmptyBlocksAndExecutedBlockNotification(t *testing.T) }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1554,7 +1554,7 @@ func Test_Selection_ProposeEmptyBlocksAndExecutedBlockNotification(t *testing.T) func Test_Selection_WithRemovingProposedBlocks(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -1583,12 +1583,12 @@ func Test_Selection_WithRemovingProposedBlocks(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1688,7 +1688,7 @@ func Test_Selection_WithRemovingProposedBlocks(t *testing.T) { func Test_SimulateSelection_ShouldNotRemoveProposedBlocks(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -1717,12 +1717,12 @@ func Test_SimulateSelection_ShouldNotRemoveProposedBlocks(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1805,7 +1805,7 @@ func Test_SimulateSelection_ShouldNotRemoveProposedBlocks(t *testing.T) { func Test_Selection_MaxTrackedBlocksReached(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(txcache.ConfigSourceMe{ Name: "test", NumChunks: 16, @@ -1843,12 +1843,12 @@ func Test_Selection_MaxTrackedBlocksReached(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -1973,7 +1973,7 @@ func Test_Selection_MaxTrackedBlocksReached(t *testing.T) { func Test_SelectionWhenFeeExceedsBalanceWithMax3TxsSelected(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -2002,12 +2002,12 @@ func Test_SelectionWhenFeeExceedsBalanceWithMax3TxsSelected(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -2131,7 +2131,7 @@ func Test_SelectionWhenFeeExceedsBalanceWithMax3TxsSelected(t *testing.T) { func Test_SelectionWhenFeeExceedsBalanceWithMax2TxsSelected(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -2160,12 +2160,12 @@ func Test_SelectionWhenFeeExceedsBalanceWithMax2TxsSelected(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -2289,7 +2289,7 @@ func Test_SelectionWhenFeeExceedsBalanceWithMax2TxsSelected(t *testing.T) { func Test_SelectionWithRootHashMismatch(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(txcache.ConfigSourceMe{ Name: "test", NumChunks: 16, @@ -2327,7 +2327,7 @@ func Test_SelectionWithRootHashMismatch(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) // keep the same root hash with the one used on the OnExecutedBlock to avoid root hash mismatch on selection selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil @@ -2374,7 +2374,7 @@ func Test_SelectionWithRootHashMismatch(t *testing.T) { func Test_SelectionWithAliceRelayerAndSenderOnSameTxs(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -2390,13 +2390,13 @@ func Test_SelectionWithAliceRelayerAndSenderOnSameTxs(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) // keep the same root hash with the one used on the OnExecutedBlock to avoid root hash mismatch on selection selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -2484,7 +2484,7 @@ func Test_SelectionWithAliceRelayerAndSenderOnSameTxs(t *testing.T) { func Test_SelectionWithAliceSenderAndThenRelayerOnDifferentTxs(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -2504,13 +2504,13 @@ func Test_SelectionWithAliceSenderAndThenRelayerOnDifferentTxs(t *testing.T) { }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) // keep the same root hash with the one used on the OnExecutedBlock to avoid root hash mismatch on selection selectionSession.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } @@ -2747,7 +2747,7 @@ func TestMempoolWithChainSimulator_Selection_InstantChangeGuardian(t *testing.T) func TestMempoolWithChainSimulator_Selection_InstantChangeGuardian_ReplaceHeader(t *testing.T) { t.Parallel() - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -2770,7 +2770,7 @@ func TestMempoolWithChainSimulator_Selection_InstantChangeGuardian_ReplaceHeader }, } - selectionSession := txcachemocks.NewSelectionSessionMockWithAccounts(accounts) + selectionSession := mempool.NewSelectionSessionMockWithAccounts(accounts) // all transactions are correctly guarded, except the last one selectionSession.IsGuardedCalled = func(tx data.TransactionHandler) bool { return true @@ -2783,7 +2783,7 @@ func TestMempoolWithChainSimulator_Selection_InstantChangeGuardian_ReplaceHeader return []byte(testRootHash), nil } - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMockWithAccounts(accounts) accountsProvider.GetRootHashCalled = func() ([]byte, error) { return []byte(testRootHash), nil } diff --git a/integrationTests/chainSimulator/mempool/testutils_test.go b/integrationTests/chainSimulator/mempool/testutils_test.go index 7d0a5ddb3b5..3b03552fff0 100644 --- a/integrationTests/chainSimulator/mempool/testutils_test.go +++ b/integrationTests/chainSimulator/mempool/testutils_test.go @@ -23,7 +23,7 @@ import ( "github.com/multiversx/mx-chain-go/node/chainSimulator/dtos" "github.com/multiversx/mx-chain-go/process/block/preprocess" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" "github.com/multiversx/mx-chain-go/txcache" ) @@ -326,7 +326,7 @@ func testOnProposed(t *testing.T, sw *core.StopWatch, numTxs int, numAddresses i // create some fake address for each account accounts := createFakeAddresses(numAddresses) - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -339,7 +339,7 @@ func testOnProposed(t *testing.T, sw *core.StopWatch, numTxs int, numAddresses i _ = initialAmount.Mul(numTxsAsBigInt, core.SafeMul(uint64(gasLimit), uint64(gasPrice))) _ = initialAmount.Add(initialAmount, core.SafeMul(uint64(numTxs), uint64(transferredValue))) - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, @@ -348,7 +348,7 @@ func testOnProposed(t *testing.T, sw *core.StopWatch, numTxs int, numAddresses i }, } - accountsAdapter := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsAdapter := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, @@ -396,7 +396,7 @@ func testFirstSelection(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToBe // create some fake address for each account accounts := createFakeAddresses(numAddresses) - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -409,7 +409,7 @@ func testFirstSelection(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToBe _ = initialAmount.Mul(numTxsAsBigInt, core.SafeMul(uint64(gasLimit), uint64(gasPrice))) _ = initialAmount.Add(initialAmount, big.NewInt(int64(numTxs))) - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, @@ -445,7 +445,7 @@ func testSecondSelection(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToB // create some fake address for each account accounts := createFakeAddresses(numAddresses) - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -458,7 +458,7 @@ func testSecondSelection(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToB _ = initialAmount.Mul(numTxsAsBigInt, core.SafeMul(uint64(gasLimit), uint64(gasPrice))) _ = initialAmount.Add(initialAmount, core.SafeMul(uint64(numTxs), uint64(transferredValue))) - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, @@ -467,7 +467,7 @@ func testSecondSelection(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToB }, } - accountsAdapter := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsAdapter := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, @@ -536,7 +536,7 @@ func testSecondSelection(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToB func testSecondSelectionWithManyTxsInPool(t *testing.T, sw *core.StopWatch, numTxs int, numTxsToBeSelected int, numAddresses int) { accounts := createFakeAddresses(numAddresses) - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() txpool, err := txcache.NewTxCache(configSourceMe, host, 0) require.Nil(t, err) @@ -549,7 +549,7 @@ func testSecondSelectionWithManyTxsInPool(t *testing.T, sw *core.StopWatch, numT _ = initialAmount.Mul(numTxsAsBigInt, core.SafeMul(uint64(gasLimit), uint64(gasPrice))) _ = initialAmount.Add(initialAmount, core.SafeMul(uint64(numTxs), uint64(transferredValue))) - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, @@ -558,7 +558,7 @@ func testSecondSelectionWithManyTxsInPool(t *testing.T, sw *core.StopWatch, numT }, } - accountsAdapter := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsAdapter := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, initialAmount, true, nil }, diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 38d97376868..28b2bf32718 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -830,11 +830,12 @@ func (tfn *TestFullNode) initInterceptors( HeartbeatExpiryTimespanInSec: 30, PeerAuthenticationTimeBetweenSendsInSec: 60, MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeout: 10 * time.Second,MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), - FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), - HardforkTrigger: &testscommon.HardforkTriggerStub{}, - NodeOperationMode: common.NormalOperation, - InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), + TrieNodeChunksInactivityTimeout: 10 * time.Second, + MainPeerShardMapper: mock.NewNetworkShardingCollectorMock(), + FullArchivePeerShardMapper: mock.NewNetworkShardingCollectorMock(), + HardforkTrigger: &testscommon.HardforkTriggerStub{}, + NodeOperationMode: common.NormalOperation, + InterceptedDataVerifierFactory: interceptorsFactory.NewInterceptedDataVerifierFactory(interceptorDataVerifierArgs), Config: config.Config{ InterceptedDataVerifier: config.InterceptedDataVerifierConfig{ CacheSpanInSec: 1, @@ -1049,7 +1050,7 @@ func (tpn *TestFullNode) initBlockProcessor( }, }, BlockTracker: tpn.BlockTracker, - MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, @@ -1431,7 +1432,7 @@ func (tpn *TestFullNode) initBlockProcessorWithSync( }, }, BlockTracker: tpn.BlockTracker, - MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, diff --git a/node/external/transactionAPI/apiTransactionProcessor.go b/node/external/transactionAPI/apiTransactionProcessor.go index e5a6286e2dd..6b3f31815e6 100644 --- a/node/external/transactionAPI/apiTransactionProcessor.go +++ b/node/external/transactionAPI/apiTransactionProcessor.go @@ -597,16 +597,16 @@ func (atp *apiTransactionProcessor) selectTransactions(accountsAdapter state.Acc return nil, err } - return atp.extractTransactions(selectedTxs, selectionOptions), nil + // selection done from outgoing txPool + return atp.extractTransactions(selectedTxs, selectionOptions, transaction.TxTypeNormal), nil } -func (atp *apiTransactionProcessor) extractTransactions(txs []*txcache.WrappedTransaction, selectionOptions common.TxSelectionOptionsAPI) []common.Transaction { +func (atp *apiTransactionProcessor) extractTransactions(txs []*txcache.WrappedTransaction, selectionOptions common.TxSelectionOptionsAPI, txType transaction.TxType) []common.Transaction { requestedFieldsHandler := newFieldsHandler(selectionOptions.GetRequestedFields()) transactions := make([]common.Transaction, len(txs)) for i, tx := range txs { - transactions[i] = atp.extractRequestedTxInfo(tx, requestedFieldsHandler) - + transactions[i] = atp.extractRequestedTxInfo(tx, requestedFieldsHandler, txType) } return transactions diff --git a/node/external/transactionAPI/apiTransactionProcessor_test.go b/node/external/transactionAPI/apiTransactionProcessor_test.go index 5abf818c1a8..c76fd90c648 100644 --- a/node/external/transactionAPI/apiTransactionProcessor_test.go +++ b/node/external/transactionAPI/apiTransactionProcessor_test.go @@ -43,7 +43,7 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" "github.com/multiversx/mx-chain-go/txcache" ) @@ -1128,7 +1128,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) require.NoError(t, err) @@ -1149,7 +1149,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolForSender(t *testing.T) { MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) require.NoError(t, err) txCacheWithMeta.AddTx(createTx(txHash3, sender, 4)) @@ -1241,7 +1241,7 @@ func TestApiTransactionProcessor_GetLastPoolNonceForSender(t *testing.T) { MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) txCacheIntraShard.AddTx(createTx(txHash2, sender, 3)) txCacheIntraShard.AddTx(createTx(txHash0, sender, 1)) @@ -1298,7 +1298,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolNonceGapsForSender(t *testin MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) require.NoError(t, err) @@ -1314,7 +1314,7 @@ func TestApiTransactionProcessor_GetTransactionsPoolNonceGapsForSender(t *testin MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) require.NoError(t, err) @@ -1406,7 +1406,7 @@ func TestApiTransactionProcessor_GetSelectedTransactions(t *testing.T) { MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) require.NoError(t, err) @@ -1746,7 +1746,7 @@ func TestApiTransactionProcessor_GetVirtualNonce(t *testing.T) { MaxNumBytesPerSenderUpperBound: 33_554_432, MaxTrackedBlocks: maxTrackedBlocks, }, - }, txcachemocks.NewMempoolHostMock(), 0) + }, mempool.NewMempoolHostMock(), 0) require.NoError(t, err) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 45539a709c1..f0fc53ca121 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -263,6 +263,7 @@ func NewBaseProcessor(arguments ArgBaseProcessor) (*baseProcessor, error) { maxProposalNonceGap: maxProposalNonceGap, ewlResetThreshold: ewlResetThreshold, closingNodeStarted: arguments.CoreComponents.ClosingNodeStarted(), + miniBlockTracker: arguments.MiniBlockTracker, } err = base.OnExecutedBlock(genesisHdr, genesisHdr.GetRootHash()) @@ -1174,52 +1175,9 @@ func isPartiallyExecuted( // check if header has the same mini blocks as presented in body func (bp *baseProcessor) checkHeaderBodyCorrelationProposal(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, blockShardID uint32) error { - mbHashesFromHdr := make(map[string]data.MiniBlockHeaderHandler, len(miniBlockHeaders)) - for i := 0; i < len(miniBlockHeaders); i++ { - if miniBlockHeaders[i] == nil { - return process.ErrNilMiniBlockHeader - } - - mbHashesFromHdr[string(miniBlockHeaders[i].GetHash())] = miniBlockHeaders[i] - } - - if len(miniBlockHeaders) != len(body.MiniBlocks) { - return process.ErrHeaderBodyMismatch - } - - if len(mbHashesFromHdr) != len(miniBlockHeaders) { - return process.ErrDuplicatedHashInBlock - } - - var mbHdr data.MiniBlockHeaderHandler - var miniBlock *block.MiniBlock - for i := 0; i < len(body.MiniBlocks); i++ { - miniBlock = body.MiniBlocks[i] - mbHdr = miniBlockHeaders[i] - if miniBlock == nil { - return process.ErrNilMiniBlock - } - if mbHdr == nil { - return process.ErrNilMiniBlockHeader - } - - mbHash, err := core.CalculateHash(bp.marshalizer, bp.hasher, miniBlock) - if err != nil { - return err - } - - mbHashStr := string(mbHash) - _, ok := mbHashesFromHdr[mbHashStr] - if !ok { - return process.ErrHeaderBodyMismatch - } - - err = bp.checkMiniBlockWithMiniBlockHeader(mbHash, mbHdr, miniBlock, blockShardID) - if err != nil { - return err - } - - delete(mbHashesFromHdr, mbHashStr) + err := bp.checkHeaderBodyCorrelation(miniBlockHeaders, body, blockShardID) + if err != nil { + return err } return bp.checkMiniBlocksConstructionProposal(miniBlockHeaders) @@ -1278,14 +1236,14 @@ func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeader(mbHash []byte, mbHdr } // check if header has the same mini blocks as presented in body -func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body) error { - mbHashesFromHdr := make(map[string]struct{}, len(miniBlockHeaders)) +func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, blockShardID uint32) error { + mbHashesFromHdr := make(map[string]data.MiniBlockHeaderHandler, len(miniBlockHeaders)) for i := 0; i < len(miniBlockHeaders); i++ { if miniBlockHeaders[i] == nil { return process.ErrNilMiniBlockHeader } - mbHashesFromHdr[string(miniBlockHeaders[i].GetHash())] = struct{}{} + mbHashesFromHdr[string(miniBlockHeaders[i].GetHash())] = miniBlockHeaders[i] } if len(miniBlockHeaders) != len(body.MiniBlocks) { @@ -1306,6 +1264,9 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini if miniBlock == nil { return process.ErrNilMiniBlock } + if mbHdr == nil { + return process.ErrNilMiniBlockHeader + } mbHash, err = core.CalculateHash(bp.marshalizer, bp.hasher, miniBlock) if err != nil { @@ -1318,7 +1279,7 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini return process.ErrHeaderBodyMismatch } - err = bp.checkMiniBlockWithMiniBlockHeader(mbHash, mbHdr, miniBlock) + err = bp.checkMiniBlockWithMiniBlockHeader(mbHash, mbHdr, miniBlock, blockShardID) if err != nil { return err } diff --git a/process/block/metablockProposal.go b/process/block/metablockProposal.go index ce9a5cdd114..8d1e9fcbd0e 100644 --- a/process/block/metablockProposal.go +++ b/process/block/metablockProposal.go @@ -247,7 +247,7 @@ func (mp *metaProcessor) VerifyBlockProposal( } } - err = mp.checkHeaderBodyCorrelationProposal(header.GetMiniBlockHeaderHandlers(), body) + err = mp.checkHeaderBodyCorrelationProposal(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) if err != nil { return err } diff --git a/process/block/shardblockProposal.go b/process/block/shardblockProposal.go index ea6eea8b3b9..13d680be81b 100644 --- a/process/block/shardblockProposal.go +++ b/process/block/shardblockProposal.go @@ -179,7 +179,7 @@ func (sp *shardProcessor) VerifyBlockProposal( return process.ErrWrongTypeAssertion } - err = sp.checkHeaderBodyCorrelationProposal(header.GetMiniBlockHeaderHandlers(), body) + err = sp.checkHeaderBodyCorrelationProposal(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) if err != nil { return err } diff --git a/process/factory/interceptorscontainer/args.go b/process/factory/interceptorscontainer/args.go index 0d154e2c0c3..050df66cc2a 100644 --- a/process/factory/interceptorscontainer/args.go +++ b/process/factory/interceptorscontainer/args.go @@ -5,8 +5,8 @@ import ( crypto "github.com/multiversx/mx-chain-crypto-go" - "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/heartbeat" "github.com/multiversx/mx-chain-go/process" @@ -51,5 +51,5 @@ type CommonInterceptorsContainerFactoryArgs struct { HardforkTrigger heartbeat.HardforkTrigger NodeOperationMode common.NodeOperation InterceptedDataVerifierFactory process.InterceptedDataVerifierFactory - Config config.Config + Config config.Config } diff --git a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go index d5933b68eb3..8feccf3127b 100644 --- a/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/metaInterceptorsContainerFactory.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/factory" @@ -135,7 +136,7 @@ func NewMetaInterceptorsContainerFactory( nodeOperationMode: args.NodeOperationMode, interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, enableEpochsHandler: args.CoreComponents.EnableEpochsHandler(), - config: args.Config, + config: args.Config, } icf := &metaInterceptorsContainerFactory{ diff --git a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go index 4fbdd56d990..e0b95b3f87d 100644 --- a/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go +++ b/process/factory/interceptorscontainer/shardInterceptorsContainerFactory.go @@ -136,7 +136,7 @@ func NewShardInterceptorsContainerFactory( nodeOperationMode: args.NodeOperationMode, interceptedDataVerifierFactory: args.InterceptedDataVerifierFactory, enableEpochsHandler: args.CoreComponents.EnableEpochsHandler(), - config: args.Config, + config: args.Config, } icf := &shardInterceptorsContainerFactory{ diff --git a/process/interceptors/processor/trieNodeChunksProcessor.go b/process/interceptors/processor/trieNodeChunksProcessor.go index 7a977e959da..8d3cc95272a 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor.go +++ b/process/interceptors/processor/trieNodeChunksProcessor.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/interceptors/processor/chunk" diff --git a/testscommon/dataRetriever/poolFactory.go b/testscommon/dataRetriever/poolFactory.go index 7e8b8e0699c..03929792a9e 100644 --- a/testscommon/dataRetriever/poolFactory.go +++ b/testscommon/dataRetriever/poolFactory.go @@ -16,7 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" "github.com/multiversx/mx-chain-go/trie/factory" ) @@ -41,7 +41,7 @@ func CreateTxPool(numShards uint32, selfShard uint32) (dataRetriever.ShardedData }, NumberOfShards: numShards, SelfShardID: selfShard, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, TxCacheBoundsConfig: config.TxCacheBoundsConfig{ MaxNumBytesPerSenderUpperBound: 33_554_432, diff --git a/testscommon/dataRetriever/poolsHolderMock.go b/testscommon/dataRetriever/poolsHolderMock.go index 903cc92fb3e..5aa28f7c430 100644 --- a/testscommon/dataRetriever/poolsHolderMock.go +++ b/testscommon/dataRetriever/poolsHolderMock.go @@ -16,7 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/storage/storageunit" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" ) // PoolsHolderMock - @@ -55,7 +55,7 @@ func NewPoolsHolderMock() *PoolsHolderMock { SizeInBytesPerSender: 10000000, Shards: 16, }, - TxGasHandler: txcachemocks.NewTxGasHandlerMock(), + TxGasHandler: mempool.NewTxGasHandlerMock(), Marshalizer: &marshal.GogoProtoMarshalizer{}, NumberOfShards: 1, TxCacheBoundsConfig: config.TxCacheBoundsConfig{ diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index c3ec2bfd0f8..c6da668dc98 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -522,14 +522,14 @@ func getLRUCacheConfig() config.CacheConfig { // GetDefaultAntifloodConfig - func GetDefaultAntifloodConfig() config.AntifloodConfig { return config.AntifloodConfig{ - Enabled: true, + Enabled: true, + MaxAllowedTrieNodeChunks: 10, + TrieNodeChunksInactivityTimeoutInSec: 10, ConfigsByRound: []config.AntifloodConfigByRound{ { Round: 0, NumConcurrentResolverJobs: 10, NumConcurrentResolvingTrieNodesJobs: 3, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeoutInSec: 10, Cache: config.CacheConfig{ Type: "LRU", Capacity: 10, @@ -600,9 +600,7 @@ func GetDefaultAntifloodConfig() config.AntifloodConfig { { Round: 100, NumConcurrentResolverJobs: 10, - NumConcurrentResolvingTrieNodesJobs: 3, - MaxAllowedTrieNodeChunks: 10, - TrieNodeChunksInactivityTimeoutInSec: 10, + NumConcurrentResolvingTrieNodesJobs: 3, Cache: config.CacheConfig{ Type: "LRU", Capacity: 10, diff --git a/testscommon/txcachemocks/accountNonceAndBalanceProviderMock.go b/testscommon/txcachemocks/mempool/accountNonceAndBalanceProviderMock.go similarity index 99% rename from testscommon/txcachemocks/accountNonceAndBalanceProviderMock.go rename to testscommon/txcachemocks/mempool/accountNonceAndBalanceProviderMock.go index 522b5b75346..2cd8704db90 100644 --- a/testscommon/txcachemocks/accountNonceAndBalanceProviderMock.go +++ b/testscommon/txcachemocks/mempool/accountNonceAndBalanceProviderMock.go @@ -1,10 +1,11 @@ -package txcachemocks +package mempool import ( "math/big" "sync" "github.com/multiversx/mx-chain-core-go/core/check" + stateMock "github.com/multiversx/mx-chain-go/testscommon/state" ) diff --git a/testscommon/txcachemocks/mempoolHostMock.go b/testscommon/txcachemocks/mempool/mempoolHostMock.go similarity index 99% rename from testscommon/txcachemocks/mempoolHostMock.go rename to testscommon/txcachemocks/mempool/mempoolHostMock.go index e90d7115d1a..f7e905f9799 100644 --- a/testscommon/txcachemocks/mempoolHostMock.go +++ b/testscommon/txcachemocks/mempool/mempoolHostMock.go @@ -1,4 +1,4 @@ -package txcachemocks +package mempool import ( "math/big" diff --git a/testscommon/txcachemocks/selectionSessionMock.go b/testscommon/txcachemocks/mempool/selectionSessionMock.go similarity index 99% rename from testscommon/txcachemocks/selectionSessionMock.go rename to testscommon/txcachemocks/mempool/selectionSessionMock.go index ba8c7aa51eb..892cfbcbc43 100644 --- a/testscommon/txcachemocks/selectionSessionMock.go +++ b/testscommon/txcachemocks/mempool/selectionSessionMock.go @@ -1,4 +1,4 @@ -package txcachemocks +package mempool import ( "math/big" @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" + stateMock "github.com/multiversx/mx-chain-go/testscommon/state" ) diff --git a/testscommon/txcachemocks/txGasHandlerMock.go b/testscommon/txcachemocks/mempool/txGasHandlerMock.go similarity index 99% rename from testscommon/txcachemocks/txGasHandlerMock.go rename to testscommon/txcachemocks/mempool/txGasHandlerMock.go index a624e29372a..afec960bab0 100644 --- a/testscommon/txcachemocks/txGasHandlerMock.go +++ b/testscommon/txcachemocks/mempool/txGasHandlerMock.go @@ -1,4 +1,4 @@ -package txcachemocks +package mempool import ( "math/big" diff --git a/txcache/autoClean_test.go b/txcache/autoClean_test.go index 8c7b05aed76..f43cf29c847 100644 --- a/txcache/autoClean_test.go +++ b/txcache/autoClean_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -112,7 +114,7 @@ func TestTxCache_Cleanup(t *testing.T) { t.Run("with GetAccountNonce errors", func(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.GetAccountNonceCalled = func(address []byte) (uint64, bool, error) { switch string(address) { case "alice": @@ -147,7 +149,7 @@ func TestTxCache_Cleanup(t *testing.T) { t.Run("with nonce equal 0", func(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 0) cache.AddTx(createTx([]byte("hash-alice-1"), "alice", 1)) @@ -163,7 +165,7 @@ func TestTxCache_Cleanup(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 3) accountsProvider.SetNonce([]byte("bob"), 42) @@ -186,7 +188,7 @@ func TestTxCache_Cleanup(t *testing.T) { t.Run("with cleanupLoopMaximumDuration cap reached", func(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 4) accountsProvider.SetNonce([]byte("bob"), 43) accountsProvider.SetNonce([]byte("carol"), 9) @@ -211,7 +213,7 @@ func TestTxCache_Cleanup(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 2) accountsProvider.SetNonce([]byte("bob"), 42) accountsProvider.SetNonce([]byte("carol"), 7) @@ -240,7 +242,7 @@ func TestTxCache_Cleanup(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 2) accountsProvider.SetNonce([]byte("bob"), 42) accountsProvider.SetNonce([]byte("carol"), 7) @@ -325,7 +327,7 @@ func TestTxCache_Cleanup(t *testing.T) { } // helper function for creating a new unconstrained cache with a given size -func newTxPoolWithN(size int, accountsProvider *txcachemocks.AccountNonceAndBalanceProviderMock) *TxCache { +func newTxPoolWithN(size int, accountsProvider *mempool.AccountNonceAndBalanceProviderMock) *TxCache { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) for i := 0; i < size; i++ { @@ -340,7 +342,7 @@ func BenchmarkAddressShuffling(b *testing.B) { for _, size := range sizes { b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { // prepare pool - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() cache := newTxPoolWithN(size, accountsProvider) b.ResetTimer() @@ -359,7 +361,7 @@ func BenchmarkCleanup(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() cache := newTxPoolWithN(size, accountsProvider) b.StartTimer() diff --git a/txcache/eviction_test.go b/txcache/eviction_test.go index 00d7ea447dc..958fc603dc1 100644 --- a/txcache/eviction_test.go +++ b/txcache/eviction_test.go @@ -7,7 +7,9 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/block" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/require" ) @@ -28,7 +30,7 @@ func TestTxCache_DoEviction_BecauseOfCount(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -64,7 +66,7 @@ func TestTxCache_DoEviction_BecauseOfSize(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -101,13 +103,13 @@ func TestTxCache_DoEviction_WithTrackedTxs(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) require.NotNil(t, cache) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() accountsProvider.SetNonce([]byte("alice"), 1) accountsProvider.SetNonce([]byte("bob"), 1) accountsProvider.SetNonce([]byte("carol"), 1) @@ -165,7 +167,7 @@ func TestTxCache_DoEviction_DoesNothingWhenAlreadyInProgress(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -205,7 +207,7 @@ func TestBenchmarkTxCache_DoEviction(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() sw := core.NewStopWatch() diff --git a/txcache/selectionTracker_test.go b/txcache/selectionTracker_test.go index f1f68953eac..3455dba9a4a 100644 --- a/txcache/selectionTracker_test.go +++ b/txcache/selectionTracker_test.go @@ -15,7 +15,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" ) func proposeBlocks(t *testing.T, numOfBlocks int, selectionTracker *selectionTracker, accountsProvider common.AccountNonceAndBalanceProvider) { @@ -158,7 +158,7 @@ func TestSelectionTracker_OnProposedBlockShouldErr(t *testing.T) { }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 1, big.NewInt(20), true, nil }, @@ -207,7 +207,7 @@ func TestSelectionTracker_OnProposedBlockShouldErr(t *testing.T) { }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 1, big.NewInt(20), true, nil }, @@ -262,7 +262,7 @@ func TestSelectionTracker_OnProposedBlockShouldErr(t *testing.T) { }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 1, big.NewInt(20), true, nil }, @@ -304,7 +304,7 @@ func TestSelectionTracker_OnProposedBlockShouldErr(t *testing.T) { }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, nil, false, expectedErr }, @@ -337,7 +337,7 @@ func TestSelectionTracker_OnProposedBlockShouldErr(t *testing.T) { }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetRootHashCalled: func() ([]byte, error) { return []byte("rootHash1"), nil }, @@ -362,7 +362,7 @@ func TestSelectionTracker_OnProposedBlockShouldWork(t *testing.T) { require.Nil(t, err) numOfBlocks := 20 - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() proposeBlocks(t, numOfBlocks, tracker, accountsProvider) require.Equal(t, 20, len(tracker.blocks)) @@ -376,7 +376,7 @@ func TestSelectionTracker_OnProposedBlockWhenMaxTrackedBlocksIsReached(t *testin require.Nil(t, err) numOfBlocks := 3 - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() proposeBlocks(t, numOfBlocks, tracker, accountsProvider) @@ -447,7 +447,7 @@ func TestSelectionTracker_OnProposedBlockWhenMaxTrackedBlocksIsReached(t *testin func Test_CompleteFlowShouldWork(t *testing.T) { t.Parallel() - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 11, big.NewInt(8 * 100000 * oneBillion), true, nil }, @@ -465,7 +465,7 @@ func Test_CompleteFlowShouldWork(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -569,7 +569,7 @@ func Test_CompleteFlowShouldWork(t *testing.T) { require.True(t, ok) require.Equal(t, expectedBreadcrumbs, tb.breadcrumbsByAddress) - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 11, big.NewInt(8 * 100000 * oneBillion), true, nil }, @@ -600,7 +600,7 @@ func Test_CompleteFlowShouldWork(t *testing.T) { } // update the session nonce - selectionSession = &txcachemocks.SelectionSessionMock{ + selectionSession = &mempool.SelectionSessionMock{ GetRootHashCalled: func() ([]byte, error) { return []byte("rootHash0"), nil }, @@ -667,7 +667,7 @@ func TestSelectionTracker_OnExecutedBlockShouldWork(t *testing.T) { require.Nil(t, err) numOfBlocks := 20 - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() proposeBlocks(t, numOfBlocks, tracker, accountsProvider) require.Equal(t, numOfBlocks, len(tracker.blocks)) @@ -682,7 +682,7 @@ func TestSelectionTracker_OnExecutedBlockShouldDeleteAllBlocksBelowSpecificNonce t.Parallel() txCache := newCacheToTest(maxNumBytesPerSenderUpperBoundTest, 3) - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() tracker, err := NewSelectionTracker(txCache, 0, maxTrackedBlocks) require.Nil(t, err) @@ -930,7 +930,7 @@ func TestSelectionTracker_deriveVirtualSelectionSessionShouldErr(t *testing.T) { t.Run("get roothash returns error, should error", func(t *testing.T) { expectedErr := errors.New("expected err") - session := txcachemocks.SelectionSessionMock{} + session := mempool.SelectionSessionMock{} session.GetRootHashCalled = func() ([]byte, error) { return nil, expectedErr } @@ -939,7 +939,7 @@ func TestSelectionTracker_deriveVirtualSelectionSessionShouldErr(t *testing.T) { require.Equal(t, expectedErr, actualErr) }) t.Run("cannot do simulation error on wrong nonce, returns error", func(t *testing.T) { - session := txcachemocks.SelectionSessionMock{} + session := mempool.SelectionSessionMock{} session.GetRootHashCalled = func() ([]byte, error) { return []byte("root hash"), nil } @@ -999,7 +999,7 @@ func TestSelectionTracker_deriveVirtualSelectionSessionShouldDeleteProposedBlock tracker.blocks = createDummyTrackedBlocks() require.Equal(t, 3, len(tracker.blocks)) - session := txcachemocks.SelectionSessionMock{} + session := mempool.SelectionSessionMock{} session.GetRootHashCalled = func() ([]byte, error) { return nil, nil } @@ -1019,7 +1019,7 @@ func TestSelectionTracker_deriveVirtualSelectionSessionShouldNotDeleteProposedBl require.Nil(t, err) require.Equal(t, 3, len(tracker.blocks)) - session := txcachemocks.SelectionSessionMock{} + session := mempool.SelectionSessionMock{} session.GetRootHashCalled = func() ([]byte, error) { return nil, nil } @@ -1072,7 +1072,7 @@ func TestSelectionTracker_validateTrackedBlocks(t *testing.T) { }, } - mockSelectionSession := txcachemocks.SelectionSessionMock{ + mockSelectionSession := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(20), true, nil }, @@ -1128,7 +1128,7 @@ func TestSelectionTracker_validateTrackedBlocks(t *testing.T) { }, } - mockSelectionSession := txcachemocks.SelectionSessionMock{ + mockSelectionSession := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(5), true, nil }, @@ -1184,7 +1184,7 @@ func TestSelectionTracker_validateTrackedBlocks(t *testing.T) { }, } - mockSelectionSession := txcachemocks.SelectionSessionMock{ + mockSelectionSession := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(2), true, nil }, @@ -1272,7 +1272,7 @@ func Test_isTransactionTracked(t *testing.T) { require.Nil(t, err) txCache.tracker = tracker - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 11, big.NewInt(6 * 100000 * oneBillion), true, nil }, @@ -1415,7 +1415,7 @@ func TestSelectionTracker_IsTransactionTracked(t *testing.T) { txCache.tracker = tracker - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 11, big.NewInt(6 * 100000 * oneBillion), true, nil }, @@ -1606,7 +1606,7 @@ func TestSelectionTracker_MaxUniqueAccounts(t *testing.T) { Nonce: 10, } - accProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetRootHashCalled: func() ([]byte, error) { return defaultLatestExecutedHash, nil }, @@ -1732,7 +1732,7 @@ func TestSelectionTracker_OnExecutedBlock_multipleBlocksWithSharedSender(t *test txCache.tracker = tracker aliceInitialNonce := uint64(1) - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return aliceInitialNonce, big.NewInt(8 * 100000 * oneBillion), true, nil }, @@ -1934,7 +1934,7 @@ func TestSelectionTracker_validateBreadcrumbsToleratesPredecessorDiscontinuity(t }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(1000), true, nil }, @@ -1990,7 +1990,7 @@ func TestSelectionTracker_validateBreadcrumbsToleratesPredecessorDiscontinuity(t }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(1000), true, nil }, @@ -2028,7 +2028,7 @@ func TestSelectionTracker_validateBreadcrumbsToleratesPredecessorDiscontinuity(t }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(1000), true, nil }, @@ -2101,7 +2101,7 @@ func TestSelectionTracker_validateBreadcrumbsToleratesPredecessorDiscontinuity(t }, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(1000), true, nil }, @@ -2140,7 +2140,7 @@ func TestSelectionTracker_SelectionSkipsDiscontinuousAccounts(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -2189,7 +2189,7 @@ func TestSelectionTracker_SelectionSkipsDiscontinuousAccounts(t *testing.T) { cache.tracker.latestRootHash = []byte("rootHash0") cache.tracker.latestNonce = 99 - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetRootHashCalled: func() ([]byte, error) { return []byte("rootHash0"), nil }, @@ -2234,7 +2234,7 @@ func TestSelectionTracker_RecoveryFromDiscontinuousBreadcrumbs(t *testing.T) { "bob": 0, } - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { nonce := accountNonces[string(address)] return nonce, big.NewInt(8 * 100000 * oneBillion), true, nil @@ -2256,7 +2256,7 @@ func TestSelectionTracker_RecoveryFromDiscontinuousBreadcrumbs(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -2329,7 +2329,7 @@ func TestSelectionTracker_RecoveryFromDiscontinuousBreadcrumbs(t *testing.T) { // Step 4: After the stale block is removed, alice's breadcrumbs are no longer in any tracked block // Now verify alice can be selected in the next selection - selectionSession := &txcachemocks.SelectionSessionMock{ + selectionSession := &mempool.SelectionSessionMock{ GetRootHashCalled: func() ([]byte, error) { return []byte("rootHash1"), nil }, diff --git a/txcache/selection_test.go b/txcache/selection_test.go index ca11769fb77..658184a4c80 100644 --- a/txcache/selection_test.go +++ b/txcache/selection_test.go @@ -17,7 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/config" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" ) var expectedError = errors.New("expected error") @@ -69,7 +69,7 @@ func TestTxCache_SelectTransactions(t *testing.T) { options := createMockTxSelectionOptions(math.MaxUint64, math.MaxInt) boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := &txcachemocks.SelectionSessionMock{ + session := &mempool.SelectionSessionMock{ GetRootHashCalled: func() ([]byte, error) { return nil, expectedError }, @@ -85,7 +85,7 @@ func TestTxCache_SelectTransactions_Dummy(t *testing.T) { options := createMockTxSelectionOptions(math.MaxUint64, math.MaxInt) boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) session.SetNonce([]byte("carol"), 1) @@ -120,7 +120,7 @@ func TestTxCache_SelectTransactions_Dummy(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) session.SetNonce([]byte("carol"), 3) @@ -147,7 +147,7 @@ func TestTxCache_SelectTransactionsWithBandwidth_Dummy(t *testing.T) { boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) session.SetNonce([]byte("carol"), 1) @@ -181,7 +181,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 42) session.SetNonce([]byte("carol"), 7) @@ -211,7 +211,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 42) session.SetNonce([]byte("carol"), 7) @@ -243,7 +243,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 42) session.SetNonce([]byte("carol"), 7) @@ -275,7 +275,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) cache.AddTx(createTx([]byte("hash-alice-1"), "alice", 1).withValue(big.NewInt(0))) @@ -301,7 +301,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetBalance([]byte("alice"), big.NewInt(150000000000000)) session.SetNonce([]byte("bob"), 42) @@ -330,7 +330,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 42) @@ -358,7 +358,7 @@ func TestTxCache_SelectTransactions_HandlesNotExecutableTransactions(t *testing. boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.IsIncorrectlyGuardedCalled = func(tx data.TransactionHandler) bool { @@ -388,7 +388,7 @@ func TestTxCache_SelectTransactions_WhenTransactionsAddedInReversedNonceOrder(t boundsConfig := createMockTxBoundsConfig() cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() // Add "nSenders" * "nTransactionsPerSender" transactions in the cache (in reversed nonce order) nSenders := 1000 @@ -427,7 +427,7 @@ func TestTxCache_SelectTransactions_WhenTransactionsAddedInReversedNonceOrder(t func TestTxCache_selectTransactionsFromBunches(t *testing.T) { t.Run("empty cache", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) options := createMockTxSelectionOptions(10_000_000_000, math.MaxInt) selected, accumulatedGas := selectTransactionsFromBunches(virtualSession, []bunchOfTransactions{}, options, 0) @@ -450,7 +450,7 @@ func TestBenchmarkTxCache_acquireBunchesOfTransactions(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() sw := core.NewStopWatch() @@ -545,7 +545,7 @@ func TestBenchmarkTxCache_selectTransactionsFromBunches(t *testing.T) { t.Run("numSenders = 1000, numTransactions = 1000", func(t *testing.T) { options := createMockTxSelectionOptions(10_000_000_000, math.MaxInt) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) bunches := createBunchesOfTransactionsWithUniformDistribution(1000, 1000) @@ -559,7 +559,7 @@ func TestBenchmarkTxCache_selectTransactionsFromBunches(t *testing.T) { t.Run("numSenders = 10000, numTransactions = 100", func(t *testing.T) { options := createMockTxSelectionOptions(10_000_000_000, math.MaxInt) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) bunches := createBunchesOfTransactionsWithUniformDistribution(1000, 1000) @@ -573,7 +573,7 @@ func TestBenchmarkTxCache_selectTransactionsFromBunches(t *testing.T) { t.Run("numSenders = 100000, numTransactions = 3", func(t *testing.T) { options := createMockTxSelectionOptions(10_000_000_000, math.MaxInt) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) bunches := createBunchesOfTransactionsWithUniformDistribution(100000, 3) @@ -589,7 +589,7 @@ func TestBenchmarkTxCache_selectTransactionsFromBunches(t *testing.T) { t.Run("numSenders = 300000, numTransactions = 1", func(t *testing.T) { options := createMockTxSelectionOptions(10_000_000_000, math.MaxInt) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) bunches := createBunchesOfTransactionsWithUniformDistribution(300000, 1) @@ -623,7 +623,7 @@ func TestBenchmarkTxCache_selectTransactionsFromBunches(t *testing.T) { func TestTxCache_selectTransactionsFromBunches_loopBreaks_whenTakesTooLong(t *testing.T) { t.Run("numSenders = 300000, numTransactions = 1", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) options := createMockTxSelectionOptionsWithTimeFunc(10_000_000_000, 50_000, haveTimeFalseForSelection) bunches := createBunchesOfTransactionsWithUniformDistribution(300000, 1) @@ -648,8 +648,8 @@ func TestBenchmarkTxCache_doSelectTransactions(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() - session := txcachemocks.NewSelectionSessionMock() + host := mempool.NewMempoolHostMock() + session := mempool.NewSelectionSessionMock() sw := core.NewStopWatch() @@ -928,7 +928,7 @@ func TestTxCache_PropagationGracePeriod(t *testing.T) { PropagationGracePeriodMs: 0, } cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) @@ -951,7 +951,7 @@ func TestTxCache_PropagationGracePeriod(t *testing.T) { PropagationGracePeriodMs: 500, } cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) @@ -974,7 +974,7 @@ func TestTxCache_PropagationGracePeriod(t *testing.T) { PropagationGracePeriodMs: 500, } cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) @@ -1001,7 +1001,7 @@ func TestTxCache_PropagationGracePeriod(t *testing.T) { PropagationGracePeriodMs: 500, } cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) cache.AddTx(createRelayedTx([]byte("hash-alice-1"), "alice", "relayer", 1)) @@ -1029,7 +1029,7 @@ func TestTxCache_PropagationGracePeriod(t *testing.T) { PropagationGracePeriodMs: 200, } cache := newUnconstrainedCacheToTest(boundsConfig) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.SetNonce([]byte("alice"), 1) session.SetNonce([]byte("bob"), 5) diff --git a/txcache/testutils_test.go b/txcache/testutils_test.go index 645eb5fc1c5..c59708bbbcd 100644 --- a/txcache/testutils_test.go +++ b/txcache/testutils_test.go @@ -10,7 +10,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" ) const oneMilion = 1000000 @@ -158,7 +159,7 @@ func addManyTransactionsWithUniformDistribution(cache *TxCache, nSenders int, nT func createBunchesOfTransactionsWithUniformDistribution(nSenders int, nTransactionsPerSender int) []bunchOfTransactions { bunches := make([]bunchOfTransactions, 0, nSenders) - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() for senderTag := 0; senderTag < nSenders; senderTag++ { bunch := make(bunchOfTransactions, 0, nTransactionsPerSender) diff --git a/txcache/transactionsHeapItem_test.go b/txcache/transactionsHeapItem_test.go index 267d76a85b2..9131c297c31 100644 --- a/txcache/transactionsHeapItem_test.go +++ b/txcache/transactionsHeapItem_test.go @@ -4,7 +4,9 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/require" ) @@ -34,7 +36,7 @@ func TestNewTransactionsHeapItem(t *testing.T) { } func TestTransactionsHeapItem_selectTransaction(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() a := createTx([]byte("tx-1"), "alice", 42) b := createTx([]byte("tx-2"), "alice", 43) @@ -157,7 +159,7 @@ func TestTransactionsHeapItem_detectNonceDuplicate(t *testing.T) { func TestTransactionsHeapItem_detectIncorrectlyGuarded(t *testing.T) { t.Run("is correctly guarded", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) session.IsIncorrectlyGuardedCalled = func(tx data.TransactionHandler) bool { @@ -171,7 +173,7 @@ func TestTransactionsHeapItem_detectIncorrectlyGuarded(t *testing.T) { }) t.Run("is incorrectly guarded", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.IsIncorrectlyGuardedCalled = func(tx data.TransactionHandler) bool { return true } diff --git a/txcache/txCache_test.go b/txcache/txCache_test.go index bd162c046cc..17d9d00e2ae 100644 --- a/txcache/txCache_test.go +++ b/txcache/txCache_test.go @@ -13,8 +13,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/config" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/multiversx/mx-chain-storage-go/common" "github.com/multiversx/mx-chain-storage-go/types" "github.com/stretchr/testify/require" @@ -33,7 +35,7 @@ func Test_NewTxCache(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -143,7 +145,7 @@ func Test_AddTx_AppliesSizeConstraintsPerSenderForNumTransactions(t *testing.T) cache := newCacheToTest(maxNumBytesPerSenderUpperBoundTest, 3) - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 1, big.NewInt(3 * 1500000 * oneBillion), true, nil }, @@ -394,7 +396,7 @@ func Test_Keys(t *testing.T) { } func Test_AddWithEviction_UniformDistributionOfTxsPerSender(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() t.Run("numSenders = 11, numTransactions = 10, countThreshold = 100, numItemsToPreemptivelyEvict = 1", func(t *testing.T) { config := ConfigSourceMe{ @@ -572,7 +574,7 @@ func TestTxCache_GetDimensionOfTrackedBlocks(t *testing.T) { require.Nil(t, err) txCache.tracker = tracker - accountsProvider := txcachemocks.NewAccountNonceAndBalanceProviderMock() + accountsProvider := mempool.NewAccountNonceAndBalanceProviderMock() err = txCache.OnProposedBlock( []byte("hash1"), @@ -661,7 +663,7 @@ func TestBenchmarkTxCache_addManyTransactionsWithSameNonce(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() sw := core.NewStopWatch() @@ -746,7 +748,7 @@ func TestBenchmarkTxCache_addManyTransactionsInDifferentScenarios(t *testing.T) TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() sw := core.NewStopWatch() t.Run("numTransactions = 5_000 with decreasing nonce (worst case)", func(t *testing.T) { @@ -825,7 +827,7 @@ func TestBenchmarkTxCache_addManyTransactionsInDifferentScenarios(t *testing.T) func Test_ResetTracker(t *testing.T) { t.Parallel() - accountsProvider := &txcachemocks.AccountNonceAndBalanceProviderMock{ + accountsProvider := &mempool.AccountNonceAndBalanceProviderMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 11, big.NewInt(6 * 100000 * oneBillion), true, nil }, @@ -843,7 +845,7 @@ func Test_ResetTracker(t *testing.T) { TxCacheBoundsConfig: createMockTxBoundsConfig(), } - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(config, host, 0) require.Nil(t, err) @@ -891,7 +893,7 @@ func Test_ResetTracker(t *testing.T) { } func newUnconstrainedCacheToTest(boundsConfig config.TxCacheBoundsConfig) *TxCache { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(ConfigSourceMe{ Name: "test", @@ -912,7 +914,7 @@ func newUnconstrainedCacheToTest(boundsConfig config.TxCacheBoundsConfig) *TxCac } func newCacheToTest(numBytesPerSenderThreshold uint32, countPerSenderThreshold uint32) *TxCache { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() cache, err := NewTxCache(ConfigSourceMe{ Name: "test", diff --git a/txcache/virtualSelectionSession_test.go b/txcache/virtualSelectionSession_test.go index 7c50ad67e77..5da2109525a 100644 --- a/txcache/virtualSelectionSession_test.go +++ b/txcache/virtualSelectionSession_test.go @@ -9,14 +9,16 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" + "github.com/stretchr/testify/require" ) func Test_newVirtualSelectionSession(t *testing.T) { t.Parallel() - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) require.NotNil(t, virtualSession) } @@ -27,7 +29,7 @@ func Test_getVirtualRecord(t *testing.T) { t.Run("should return virtual record", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{} + sessionMock := mempool.SelectionSessionMock{} virtualSession := newVirtualSelectionSession(&sessionMock, make(map[string]*virtualAccountRecord)) expectedRecord := virtualAccountRecord{ @@ -52,7 +54,7 @@ func Test_getVirtualRecord(t *testing.T) { t.Run("should return account from real session", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 2, big.NewInt(2), true, nil }, @@ -80,7 +82,7 @@ func Test_getVirtualRecord(t *testing.T) { t.Run("should create empty record when account does not exist", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(0), false, nil }, @@ -98,7 +100,7 @@ func Test_getVirtualRecord(t *testing.T) { t.Parallel() expErr := errors.New("error") - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, nil, false, expErr }, @@ -117,7 +119,7 @@ func Test_getNonce(t *testing.T) { t.Run("should return nonce from real session", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 2, big.NewInt(2), true, nil }, @@ -136,7 +138,7 @@ func Test_getNonce(t *testing.T) { t.Run("should return nonce from account record", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 2, big.NewInt(2), true, nil }, @@ -170,7 +172,7 @@ func Test_getNonce(t *testing.T) { t.Run("should err", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{} + sessionMock := mempool.SelectionSessionMock{} virtualSession := newVirtualSelectionSession(&sessionMock, make(map[string]*virtualAccountRecord)) aliceRecord, err := newVirtualAccountRecord(core.OptionalUint64{Value: 0, HasValue: false}, big.NewInt(1)) @@ -183,7 +185,7 @@ func Test_getNonce(t *testing.T) { t.Run("should return errNonceNotSet", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 2, big.NewInt(2), true, nil }, @@ -214,10 +216,10 @@ func Test_getNonce(t *testing.T) { } func Test_accumulateConsumedBalance(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() t.Run("when sender is fee payer", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) a := createTx([]byte("a-7"), "a", 7) @@ -246,7 +248,7 @@ func Test_accumulateConsumedBalance(t *testing.T) { }) t.Run("when relayer is fee payer", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) a := createTx([]byte("a-7"), "a", 7).withRelayer([]byte("b")).withGasLimit(100_000) @@ -288,7 +290,7 @@ func Test_detectWillBalanceBeExceeded(t *testing.T) { t.Run("should exceed balance", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{} + sessionMock := mempool.SelectionSessionMock{} virtualSession := newVirtualSelectionSession(&sessionMock, make(map[string]*virtualAccountRecord)) aliceRecord := virtualAccountRecord{ @@ -321,7 +323,7 @@ func Test_detectWillBalanceBeExceeded(t *testing.T) { t.Run("should not exceed balance", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{} + sessionMock := mempool.SelectionSessionMock{} virtualSession := newVirtualSelectionSession(&sessionMock, make(map[string]*virtualAccountRecord)) aliceRecord := virtualAccountRecord{ @@ -358,7 +360,7 @@ func Test_isIncorrectlyGuarded(t *testing.T) { t.Run("should return not correctly guarded", func(t *testing.T) { t.Parallel() - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ IsIncorrectlyGuardedCalled: func(tx data.TransactionHandler) bool { return true }, @@ -375,7 +377,7 @@ func TestBenchmarkVirtualSelectionSession_getRecord(t *testing.T) { sw := core.NewStopWatch() t.Run("numAccounts = 300, numTransactionsPerAccount = 100", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() virtualSession := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) numAccounts := 300 @@ -403,7 +405,7 @@ func TestBenchmarkVirtualSelectionSession_getRecord(t *testing.T) { }) t.Run("numAccounts = 10_000, numTransactionsPerAccount = 3", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() sessionWrapper := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) numAccounts := 10_000 @@ -431,7 +433,7 @@ func TestBenchmarkVirtualSelectionSession_getRecord(t *testing.T) { }) t.Run("numAccounts = 30_000, numTransactionsPerAccount = 1", func(t *testing.T) { - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() sessionWrapper := newVirtualSelectionSession(session, make(map[string]*virtualAccountRecord)) numAccounts := 30_000 @@ -481,7 +483,7 @@ func Test_setChangeGuardianIfNeeded(t *testing.T) { b := createTx([]byte("tx-2"), "alice", 43) c := createTx([]byte("tx-3"), "alice", 44).withData([]byte("SetGuardian@newGuardian")).withGasLimit(100000) - session := txcachemocks.NewSelectionSessionMock() + session := mempool.NewSelectionSessionMock() session.IsIncorrectlyGuardedCalled = func(tx data.TransactionHandler) bool { return tx.GetNonce() == b.Tx.GetNonce() // for coverage } diff --git a/txcache/virtualSessionComputer_test.go b/txcache/virtualSessionComputer_test.go index acdcb778b6a..f3f2377f8a8 100644 --- a/txcache/virtualSessionComputer_test.go +++ b/txcache/virtualSessionComputer_test.go @@ -8,7 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/stretchr/testify/require" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" ) func Test_fromBreadcrumbToVirtualRecord(t *testing.T) { @@ -97,7 +97,7 @@ func Test_createVirtualSelectionSession(t *testing.T) { t.Parallel() t.Run("should create blocked record for carol because it has discontinuous nonce with session nonce", func(t *testing.T) { - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 2, big.NewInt(2), true, nil }, @@ -211,7 +211,7 @@ func Test_createVirtualSelectionSession(t *testing.T) { t.Run("should return error from selection session", func(t *testing.T) { var expectedErr = errors.New("expected err") - sessionMock := txcachemocks.SelectionSessionMock{ + sessionMock := mempool.SelectionSessionMock{ GetAccountNonceAndBalanceCalled: func(address []byte) (uint64, *big.Int, bool, error) { return 0, big.NewInt(0), true, expectedErr }, diff --git a/txcache/wrappedTransaction_test.go b/txcache/wrappedTransaction_test.go index 12a8c517dd2..48ed857e45b 100644 --- a/txcache/wrappedTransaction_test.go +++ b/txcache/wrappedTransaction_test.go @@ -5,13 +5,14 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-go/testscommon/txcachemocks" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-go/testscommon/txcachemocks/mempool" ) func TestWrappedTransaction_precomputeFields(t *testing.T) { t.Run("only move balance gas limit", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() tx := createTx([]byte("a"), "a", 1).withValue(oneQuintillionBig).withDataLength(1).withGasLimit(51500).withGasPrice(oneBillion) tx.precomputeFields(host) @@ -23,7 +24,7 @@ func TestWrappedTransaction_precomputeFields(t *testing.T) { }) t.Run("move balance gas limit and execution gas limit (a)", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() tx := createTx([]byte("b"), "b", 1).withDataLength(1).withGasLimit(51501).withGasPrice(oneBillion) tx.precomputeFields(host) @@ -34,7 +35,7 @@ func TestWrappedTransaction_precomputeFields(t *testing.T) { }) t.Run("move balance gas limit and execution gas limit (b)", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() tx := createTx([]byte("c"), "c", 1).withDataLength(1).withGasLimit(oneMilion).withGasPrice(oneBillion) tx.precomputeFields(host) @@ -47,7 +48,7 @@ func TestWrappedTransaction_precomputeFields(t *testing.T) { }) t.Run("with guardian", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() tx := createTx([]byte("a"), "a", 1).withValue(oneQuintillionBig) tx.precomputeFields(host) @@ -59,7 +60,7 @@ func TestWrappedTransaction_precomputeFields(t *testing.T) { }) t.Run("with nil transferred value", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() tx := createTx([]byte("a"), "a", 1) tx.precomputeFields(host) @@ -69,7 +70,7 @@ func TestWrappedTransaction_precomputeFields(t *testing.T) { }) t.Run("queries host", func(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() host.ComputeTxFeeCalled = func(_ data.TransactionWithFeeHandler) *big.Int { return big.NewInt(42) } @@ -86,7 +87,7 @@ func TestWrappedTransaction_precomputeFields(t *testing.T) { } func TestWrappedTransaction_decideFeePayer(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() t.Run("when sender is fee payer", func(t *testing.T) { tx := createTx([]byte("a"), "a", 1) @@ -106,7 +107,7 @@ func TestWrappedTransaction_decideFeePayer(t *testing.T) { } func TestWrappedTransaction_isTransactionMoreValuableForNetwork(t *testing.T) { - host := txcachemocks.NewMempoolHostMock() + host := mempool.NewMempoolHostMock() t.Run("decide by price per unit", func(t *testing.T) { a := createTx([]byte("a-1"), "a", 1).withDataLength(1).withGasLimit(51500).withGasPrice(oneBillion) diff --git a/update/factory/fullSyncInterceptors.go b/update/factory/fullSyncInterceptors.go index 23545c47704..f681c27d8eb 100644 --- a/update/factory/fullSyncInterceptors.go +++ b/update/factory/fullSyncInterceptors.go @@ -22,6 +22,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/disabled" ) From 6c90a5418fe22919e9e4a559475d97b05912d0a4 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 13:44:28 +0300 Subject: [PATCH 101/116] fixes after merge --- consensus/broadcast/delayedBroadcast.go | 6 -- consensus/broadcast/delayedBroadcast_test.go | 18 ++-- .../broadcast/shardChainMessenger_test.go | 1 - consensus/spos/sposFactory/sposFactory.go | 2 - .../spos/sposFactory/sposFactory_test.go | 12 --- epochStart/metachain/trigger_test.go | 10 +- factory/consensus/consensusComponents.go | 1 - integrationTests/testFullNode.go | 1 - integrationTests/testProcessorNode.go | 4 +- .../components/testOnlyProcessingNode.go | 3 - testscommon/txcachemocks/txCacheMock.go | 96 +++++++++++++++---- 11 files changed, 98 insertions(+), 56 deletions(-) diff --git a/consensus/broadcast/delayedBroadcast.go b/consensus/broadcast/delayedBroadcast.go index cf4be200683..96f0d9cb468 100644 --- a/consensus/broadcast/delayedBroadcast.go +++ b/consensus/broadcast/delayedBroadcast.go @@ -38,7 +38,6 @@ type shardDataHandler interface { type ArgsDelayedBlockBroadcaster struct { InterceptorsContainer process.InterceptorsContainer HeadersSubscriber consensus.HeadersPoolSubscriber - HeadersPool consensus.HeadersPoolGetter ProofsPool consensus.EquivalentProofsPool EnableEpochsHandler common.EnableEpochsHandler ShardCoordinator sharding.Coordinator @@ -71,7 +70,6 @@ type delayedBlockBroadcaster struct { interceptorsContainer process.InterceptorsContainer shardCoordinator sharding.Coordinator headersSubscriber consensus.HeadersPoolSubscriber - headersPool consensus.HeadersPoolGetter proofsPool consensus.EquivalentProofsPool enableEpochsHandler common.EnableEpochsHandler valHeaderBroadcastData []*shared.ValidatorHeaderBroadcastData @@ -104,9 +102,6 @@ func NewDelayedBlockBroadcaster(args *ArgsDelayedBlockBroadcaster) (*delayedBloc if check.IfNil(args.HeadersSubscriber) { return nil, spos.ErrNilHeadersSubscriber } - if check.IfNil(args.HeadersPool) { - return nil, spos.ErrNilHeadersPool - } if check.IfNil(args.ProofsPool) { return nil, spos.ErrNilEquivalentProofPool } @@ -132,7 +127,6 @@ func NewDelayedBlockBroadcaster(args *ArgsDelayedBlockBroadcaster) (*delayedBloc shardCoordinator: args.ShardCoordinator, interceptorsContainer: args.InterceptorsContainer, headersSubscriber: args.HeadersSubscriber, - headersPool: args.HeadersPool, proofsPool: args.ProofsPool, enableEpochsHandler: args.EnableEpochsHandler, valHeaderBroadcastData: make([]*shared.ValidatorHeaderBroadcastData, 0), diff --git a/consensus/broadcast/delayedBroadcast_test.go b/consensus/broadcast/delayedBroadcast_test.go index ba377c065b2..3ed91e53236 100644 --- a/consensus/broadcast/delayedBroadcast_test.go +++ b/consensus/broadcast/delayedBroadcast_test.go @@ -193,9 +193,9 @@ func TestNewDelayedBlockBroadcaster_NilHeadersPoolShouldErr(t *testing.T) { t.Parallel() delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() - delayBroadcasterArgs.HeadersPool = nil + delayBroadcasterArgs.HeadersSubscriber = nil dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) - require.Equal(t, spos.ErrNilHeadersPool, err) + require.Equal(t, spos.ErrNilHeadersSubscriber, err) require.Nil(t, dbb) } @@ -205,7 +205,7 @@ func TestNewDelayedBlockBroadcaster_NilProofsPoolShouldErr(t *testing.T) { delayBroadcasterArgs := createDefaultDelayedBroadcasterArgs() delayBroadcasterArgs.ProofsPool = nil dbb, err := broadcast.NewDelayedBlockBroadcaster(delayBroadcasterArgs) - require.Equal(t, process.ErrNilProofsPool, err) + require.Equal(t, spos.ErrNilEquivalentProofPool, err) require.Nil(t, dbb) } @@ -281,7 +281,7 @@ func TestDelayedBlockBroadcaster_ReceivedProof_HeaderNotInPoolShouldNotBroadcast return flag == common.AndromedaFlag }, } - delayBroadcasterArgs.HeadersPool = &pool.HeadersPoolStub{ + delayBroadcasterArgs.HeadersSubscriber = &pool.HeadersPoolStub{ GetHeaderByHashCalled: func(hash []byte) (data.HeaderHandler, error) { return nil, errors.New("not found") }, @@ -310,7 +310,7 @@ func TestDelayedBlockBroadcaster_ReceivedProof_HeaderNotInPoolShouldNotBroadcast HeaderShardId: core.MetachainShardId, HeaderNonce: 1, } - dbb.ReceivedProof(proof) + dbb.ProofReceived(proof) time.Sleep(common.ExtraDelayForBroadcastBlockInfo + common.ExtraDelayBetweenBroadcastMbsAndTxs + 100*time.Millisecond) assert.False(t, mbBroadcastCalled.IsSet(), "should NOT broadcast when header is not in pool") @@ -339,7 +339,7 @@ func TestDelayedBlockBroadcaster_ReceivedProof_NonMetaShouldBeIgnored(t *testing HeaderHash: []byte("shard hash"), HeaderShardId: 0, // not metachain } - dbb.ReceivedProof(proof) + dbb.ProofReceived(proof) time.Sleep(50 * time.Millisecond) assert.False(t, mbBroadcastCalled.IsSet(), "should NOT broadcast for non-metachain proofs") @@ -353,7 +353,7 @@ func TestDelayedBlockBroadcaster_ReceivedProof_NilProofShouldNotPanic(t *testing require.Nil(t, err) require.NotPanics(t, func() { - dbb.ReceivedProof(nil) + dbb.ProofReceived(nil) }) } @@ -413,7 +413,7 @@ func TestDelayedBlockBroadcaster_HeaderArrivesFirst_ThenProofTriggersBroadcast(t metaBlock.Nonce = 1 metaHash := []byte("meta hash") - delayBroadcasterArgs.HeadersPool = &pool.HeadersPoolStub{ + delayBroadcasterArgs.HeadersSubscriber = &pool.HeadersPoolStub{ GetHeaderByHashCalled: func(hash []byte) (data.HeaderHandler, error) { if bytes.Equal(hash, metaHash) { return metaBlock, nil @@ -454,7 +454,7 @@ func TestDelayedBlockBroadcaster_HeaderArrivesFirst_ThenProofTriggersBroadcast(t HeaderShardId: core.MetachainShardId, HeaderNonce: 1, } - dbb.ReceivedProof(proof) + dbb.ProofReceived(proof) time.Sleep(common.ExtraDelayForBroadcastBlockInfo + common.ExtraDelayBetweenBroadcastMbsAndTxs + 100*time.Millisecond) assert.True(t, mbBroadcastCalled.IsSet(), "should broadcast after proof arrives") } diff --git a/consensus/broadcast/shardChainMessenger_test.go b/consensus/broadcast/shardChainMessenger_test.go index 0b0d55d62b1..c349f7fd46a 100644 --- a/consensus/broadcast/shardChainMessenger_test.go +++ b/consensus/broadcast/shardChainMessenger_test.go @@ -574,7 +574,6 @@ func TestShardChainMessenger_BroadcastBlockDataLeaderShouldTriggerWaitingDelayed argsDelayedBroadcaster := broadcast.ArgsDelayedBlockBroadcaster{ InterceptorsContainer: args.InterceptorsContainer, HeadersSubscriber: args.HeadersSubscriber, - HeadersPool: &pool.HeadersPoolStub{}, ProofsPool: &dataRetrieverMock.ProofsPoolMock{}, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ShardCoordinator: args.ShardCoordinator, diff --git a/consensus/spos/sposFactory/sposFactory.go b/consensus/spos/sposFactory/sposFactory.go index 6f2900c327b..d8dc5fb93c5 100644 --- a/consensus/spos/sposFactory/sposFactory.go +++ b/consensus/spos/sposFactory/sposFactory.go @@ -34,7 +34,6 @@ func GetBroadcastMessenger( shardCoordinator sharding.Coordinator, peerSignatureHandler crypto.PeerSignatureHandler, headersSubscriber consensus.HeadersPoolSubscriber, - headersPool consensus.HeadersPoolGetter, proofsPool consensus.EquivalentProofsPool, enableEpochsHandler common.EnableEpochsHandler, interceptorsContainer process.InterceptorsContainer, @@ -49,7 +48,6 @@ func GetBroadcastMessenger( dbbArgs := &broadcast.ArgsDelayedBlockBroadcaster{ InterceptorsContainer: interceptorsContainer, HeadersSubscriber: headersSubscriber, - HeadersPool: headersPool, ProofsPool: proofsPool, EnableEpochsHandler: enableEpochsHandler, ShardCoordinator: shardCoordinator, diff --git a/consensus/spos/sposFactory/sposFactory_test.go b/consensus/spos/sposFactory/sposFactory_test.go index 1d9dab23289..92a08e45107 100644 --- a/consensus/spos/sposFactory/sposFactory_test.go +++ b/consensus/spos/sposFactory/sposFactory_test.go @@ -59,14 +59,11 @@ func TestGetBroadcastMessenger_ShardShouldWork(t *testing.T) { shardCoord, peerSigHandler, headersSubscriber, - headersSubscriber, &dataRetrieverMock.ProofsPoolMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, - &dataRetrieverMock.ProofsPoolMock{}, - &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, err) @@ -95,14 +92,11 @@ func TestGetBroadcastMessenger_MetachainShouldWork(t *testing.T) { shardCoord, peerSigHandler, headersSubscriber, - headersSubscriber, &dataRetrieverMock.ProofsPoolMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, - &dataRetrieverMock.ProofsPoolMock{}, - &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, err) @@ -123,14 +117,11 @@ func TestGetBroadcastMessenger_NilShardCoordinatorShouldErr(t *testing.T) { nil, nil, headersSubscriber, - headersSubscriber, &dataRetrieverMock.ProofsPoolMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, - &dataRetrieverMock.ProofsPoolMock{}, - &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, bm) @@ -155,14 +146,11 @@ func TestGetBroadcastMessenger_InvalidShardIdShouldErr(t *testing.T) { shardCoord, nil, headersSubscriber, - headersSubscriber, &dataRetrieverMock.ProofsPoolMock{}, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, interceptosContainer, alarmSchedulerStub, &testscommon.KeysHandlerStub{}, - &dataRetrieverMock.ProofsPoolMock{}, - &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, ) assert.Nil(t, bm) diff --git a/epochStart/metachain/trigger_test.go b/epochStart/metachain/trigger_test.go index fed23a20624..7483fae6a99 100644 --- a/epochStart/metachain/trigger_test.go +++ b/epochStart/metachain/trigger_test.go @@ -314,8 +314,14 @@ func TestTrigger_ForceEpochStartShouldWaitMinimumNonceEvenWhenForced(t *testing. t.Parallel() arguments := createMockEpochStartTriggerArguments() - arguments.Settings.MinRoundsBetweenEpochs = 20 - arguments.Settings.RoundsPerEpoch = 200 + arguments.ChainParametersHandler = &chainParameters.ChainParametersHandlerStub{ + ChainParametersForEpochCalled: func(epoch uint32) (config.ChainParametersByEpochConfig, error) { + return config.ChainParametersByEpochConfig{ + MinRoundsBetweenEpochs: 20, + RoundsPerEpoch: 200, + }, nil + }, + } epochStartTrigger, err := NewEpochStartTrigger(arguments) require.Nil(t, err) diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index 9f846994b3a..863f772e28b 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -161,7 +161,6 @@ func (ccf *consensusComponentsFactory) Create() (*consensusComponents, error) { ccf.processComponents.ShardCoordinator(), ccf.cryptoComponents.PeerSignatureHandler(), ccf.dataComponents.Datapool().Headers(), - ccf.dataComponents.Datapool().Headers(), ccf.dataComponents.Datapool().Proofs(), ccf.coreComponents.EnableEpochsHandler(), ccf.processComponents.InterceptorsContainer(), diff --git a/integrationTests/testFullNode.go b/integrationTests/testFullNode.go index 28b2bf32718..ae6f2f9d362 100644 --- a/integrationTests/testFullNode.go +++ b/integrationTests/testFullNode.go @@ -394,7 +394,6 @@ func (tpn *TestFullNode) initTestNodeWithArgs(args ArgTestProcessorNode, fullArg tpn.ShardCoordinator, tpn.OwnAccount.PeerSigHandler, tpn.DataPool.Headers(), - tpn.DataPool.Headers(), tpn.DataPool.Proofs(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tpn.MainInterceptorsContainer, diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index e3b4e0b3115..24dd8c40045 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -957,7 +957,6 @@ func (tpn *TestProcessorNode) initTestNodeWithArgs(args ArgTestProcessorNode) { tpn.ShardCoordinator, tpn.OwnAccount.PeerSigHandler, tpn.DataPool.Headers(), - tpn.DataPool.Headers(), tpn.DataPool.Proofs(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tpn.MainInterceptorsContainer, @@ -1190,7 +1189,6 @@ func (tpn *TestProcessorNode) InitializeProcessors(gasMap map[string]map[string] tpn.ShardCoordinator, tpn.OwnAccount.PeerSigHandler, tpn.DataPool.Headers(), - tpn.DataPool.Headers(), tpn.DataPool.Proofs(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tpn.MainInterceptorsContainer, @@ -2646,7 +2644,7 @@ func (tpn *TestProcessorNode) initBlockProcessor() { }, }, BlockTracker: tpn.BlockTracker, - MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: TestBlockSizeThrottler, HistoryRepository: tpn.HistoryRepository, GasHandler: tpn.GasHandler, diff --git a/node/chainSimulator/components/testOnlyProcessingNode.go b/node/chainSimulator/components/testOnlyProcessingNode.go index fde1fd8abea..b0753f468cf 100644 --- a/node/chainSimulator/components/testOnlyProcessingNode.go +++ b/node/chainSimulator/components/testOnlyProcessingNode.go @@ -374,14 +374,11 @@ func (node *testOnlyProcessingNode) createBroadcastMessenger() error { node.ProcessComponentsHolder.ShardCoordinator(), node.CryptoComponentsHolder.PeerSignatureHandler(), node.DataComponentsHolder.Datapool().Headers(), - node.DataComponentsHolder.Datapool().Headers(), node.DataComponentsHolder.Datapool().Proofs(), node.CoreComponentsHolder.EnableEpochsHandler(), node.ProcessComponentsHolder.InterceptorsContainer(), node.CoreComponentsHolder.AlarmScheduler(), node.CryptoComponentsHolder.KeysHandler(), - node.DataComponentsHolder.Datapool().Proofs(), - node.CoreComponentsHolder.EnableEpochsHandler(), ) if err != nil { return err diff --git a/testscommon/txcachemocks/txCacheMock.go b/testscommon/txcachemocks/txCacheMock.go index 755b97a334c..ff5a5e52bc3 100644 --- a/testscommon/txcachemocks/txCacheMock.go +++ b/testscommon/txcachemocks/txCacheMock.go @@ -1,24 +1,30 @@ package txcachemocks -import "github.com/multiversx/mx-chain-go/txcache" +import ( + "time" + + "github.com/multiversx/mx-chain-core-go/data" + + "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/txcache" +) // TxCacheMock - type TxCacheMock struct { - ClearCalled func() - PutCalled func(key []byte, value interface{}, sizeInBytes int) (evicted bool) - GetCalled func(key []byte) (value interface{}, ok bool) - HasCalled func(key []byte) bool - PeekCalled func(key []byte) (value interface{}, ok bool) - HasOrAddCalled func(key []byte, value interface{}, sizeInBytes int) (has, added bool) - RemoveCalled func(key []byte) - RemoveOldestCalled func() - KeysCalled func() [][]byte - LenCalled func() int - MaxSizeCalled func() int - RegisterHandlerCalled func(func(key []byte, value interface{})) - UnRegisterHandlerCalled func(id string) - CloseCalled func() error - + ClearCalled func() + PutCalled func(key []byte, value interface{}, sizeInBytes int) (evicted bool) + GetCalled func(key []byte) (value interface{}, ok bool) + HasCalled func(key []byte) bool + PeekCalled func(key []byte) (value interface{}, ok bool) + HasOrAddCalled func(key []byte, value interface{}, sizeInBytes int) (has, added bool) + RemoveCalled func(key []byte) + RemoveOldestCalled func() + KeysCalled func() [][]byte + LenCalled func() int + MaxSizeCalled func() int + RegisterHandlerCalled func(func(key []byte, value interface{})) + UnRegisterHandlerCalled func(id string) + CloseCalled func() error AddTxCalled func(tx *txcache.WrappedTransaction) (ok bool, added bool) GetByTxHashCalled func(txHash []byte) (*txcache.WrappedTransaction, bool) RemoveTxByHashCalled func(txHash []byte) bool @@ -28,6 +34,64 @@ type TxCacheMock struct { NumBytesCalled func() int DiagnoseCalled func(deep bool) GetTransactionsPoolForSenderCalled func(sender string) []*txcache.WrappedTransaction + GetTrackerDiagnosisCalled func() txcache.TrackerDiagnosis + OnProposedBlockCalled func(blockHash []byte, blockBody data.BodyHandler, blockHeader data.HeaderHandler, accountsProvider common.AccountNonceAndBalanceProvider, latestExecutedHash []byte) error + OnBackfilledBlockCalled func(blockHash []byte, blockBody data.BodyHandler, blockHeader data.HeaderHandler) error + OnExecutedBlockCalled func(blockHeader data.HeaderHandler, rootHash []byte) error + ResetTrackerCalled func() + CleanupCalled func(accountsProvider common.AccountNonceProvider, randomness uint64, maxNum int, cleanupLoopMaximumDurationMs time.Duration) uint64 +} + +// GetTrackerDiagnosis - +func (cache *TxCacheMock) GetTrackerDiagnosis() txcache.TrackerDiagnosis { + if cache.GetTrackerDiagnosisCalled != nil { + return cache.GetTrackerDiagnosisCalled() + } + + return nil +} + +// OnProposedBlock - +func (cache *TxCacheMock) OnProposedBlock(blockHash []byte, blockBody data.BodyHandler, blockHeader data.HeaderHandler, accountsProvider common.AccountNonceAndBalanceProvider, latestExecutedHash []byte) error { + if cache.OnProposedBlockCalled != nil { + return cache.OnProposedBlockCalled(blockHash, blockBody, blockHeader, accountsProvider, latestExecutedHash) + } + + return nil +} + +// OnBackfilledBlock - +func (cache *TxCacheMock) OnBackfilledBlock(blockHash []byte, blockBody data.BodyHandler, blockHeader data.HeaderHandler) error { + if cache.OnBackfilledBlockCalled != nil { + return cache.OnBackfilledBlockCalled(blockHash, blockBody, blockHeader) + } + + return nil +} + +// OnExecutedBlock - +func (cache *TxCacheMock) OnExecutedBlock(blockHeader data.HeaderHandler, rootHash []byte) error { + if cache.OnExecutedBlockCalled != nil { + return cache.OnExecutedBlockCalled(blockHeader, rootHash) + } + + return nil +} + +// ResetTracker - +func (cache *TxCacheMock) ResetTracker() { + if cache.ResetTrackerCalled != nil { + cache.ResetTrackerCalled() + } +} + +// Cleanup - +func (cache *TxCacheMock) Cleanup(accountsProvider common.AccountNonceProvider, randomness uint64, maxNum int, cleanupLoopMaximumDurationMs time.Duration) uint64 { + if cache.CleanupCalled != nil { + return cache.CleanupCalled(accountsProvider, randomness, maxNum, cleanupLoopMaximumDurationMs) + } + + return 0 } // NewTxCacheStub - From 50d76952b83f8b848e2a56a67c8d19cec52df4ca Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 14:29:31 +0300 Subject: [PATCH 102/116] fixes after merge --- process/block/baseProcess.go | 52 ++++++++++++++++------------- process/block/metablock.go | 6 ++-- process/block/metablockProposal.go | 2 +- process/block/shardblock.go | 2 +- process/block/shardblockProposal.go | 2 +- 5 files changed, 34 insertions(+), 30 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index f0fc53ca121..39df465dfb7 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1173,30 +1173,19 @@ func isPartiallyExecuted( return processedMiniBlockInfo != nil && !processedMiniBlockInfo.FullyProcessed } -// check if header has the same mini blocks as presented in body -func (bp *baseProcessor) checkHeaderBodyCorrelationProposal(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, blockShardID uint32) error { - err := bp.checkHeaderBodyCorrelation(miniBlockHeaders, body, blockShardID) - if err != nil { - return err +func (bp *baseProcessor) checkConstructionStateProcessingTypeAndIndexesCorrectnessProposal(miniBlockHeader data.MiniBlockHeaderHandler) error { + // for Supernova all miniBlocks not part of an execution result need to have construction state Proposed + if miniBlockHeader.GetConstructionState() != int32(block.Proposed) { + return process.ErrWrongMiniBlockConstructionState } - - return bp.checkMiniBlocksConstructionProposal(miniBlockHeaders) -} - -func (bp *baseProcessor) checkMiniBlocksConstructionProposal(miniBlockHeaders []data.MiniBlockHeaderHandler) error { - for i := 0; i < len(miniBlockHeaders); i++ { - // for Supernova all miniBlocks not part of an execution result need to have construction state Proposed - if miniBlockHeaders[i].GetConstructionState() != int32(block.Proposed) { - return process.ErrWrongMiniBlockConstructionState - } - if miniBlockHeaders[i].GetProcessingType() != int32(block.Normal) { - return process.ErrWrongMiniBlockProcessingType - } + if miniBlockHeader.GetProcessingType() != int32(block.Normal) { + return process.ErrWrongMiniBlockProcessingType } + return nil } -func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeader(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { +func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { if !bytes.Equal(mbHash, mbHdr.GetHash()) { return process.ErrHeaderBodyMismatch } @@ -1222,21 +1211,32 @@ func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeader(mbHash []byte, mbHdr return err } - err = checkConstructionStateProcessingTypeAndIndexesCorrectness(mbHdr, miniBlock, blockShardID) + err = bp.checkIndexOfFirstTxProcessedAgainstTracker(mbHdr, mbHash) if err != nil { return err } - err = bp.checkIndexOfFirstTxProcessedAgainstTracker(mbHdr, mbHash) + return nil +} + +func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderProposal(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { + err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock, blockShardID) if err != nil { return err } + return bp.checkConstructionStateProcessingTypeAndIndexesCorrectnessProposal(mbHdr) +} - return nil +func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeader(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { + err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock, blockShardID) + if err != nil { + return err + } + return checkConstructionStateProcessingTypeAndIndexesCorrectness(mbHdr, miniBlock, blockShardID) } // check if header has the same mini blocks as presented in body -func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, blockShardID uint32) error { +func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, blockShardID uint32, proposal bool) error { mbHashesFromHdr := make(map[string]data.MiniBlockHeaderHandler, len(miniBlockHeaders)) for i := 0; i < len(miniBlockHeaders); i++ { if miniBlockHeaders[i] == nil { @@ -1279,7 +1279,11 @@ func (bp *baseProcessor) checkHeaderBodyCorrelation(miniBlockHeaders []data.Mini return process.ErrHeaderBodyMismatch } - err = bp.checkMiniBlockWithMiniBlockHeader(mbHash, mbHdr, miniBlock, blockShardID) + if !proposal { + err = bp.checkMiniBlockWithMiniBlockHeader(mbHash, mbHdr, miniBlock, blockShardID) + } else { + err = bp.checkMiniBlockWithMiniBlockHeaderProposal(mbHash, mbHdr, miniBlock, blockShardID) + } if err != nil { return err } diff --git a/process/block/metablock.go b/process/block/metablock.go index c67222a3b68..618817b7d67 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -16,11 +16,10 @@ import ( "github.com/multiversx/mx-chain-core-go/data/headerVersionData" logger "github.com/multiversx/mx-chain-logger-go" - epochStartMetaCommmon "github.com/multiversx/mx-chain-go/epochStart/metachain" - "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/dataRetriever" + epochStartMetaCommmon "github.com/multiversx/mx-chain-go/epochStart/metachain" processOutport "github.com/multiversx/mx-chain-go/outport/process" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/process/asyncExecution/executionTrack" @@ -28,6 +27,7 @@ import ( "github.com/multiversx/mx-chain-go/process/block/helpers" "github.com/multiversx/mx-chain-go/process/block/processedMb" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/trie" ) const ( @@ -197,7 +197,7 @@ func (mp *metaProcessor) ProcessBlock( return process.ErrWrongTypeAssertion } - err = mp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) + err = mp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID(), false) if err != nil { return err } diff --git a/process/block/metablockProposal.go b/process/block/metablockProposal.go index 8d1e9fcbd0e..59f98e6c978 100644 --- a/process/block/metablockProposal.go +++ b/process/block/metablockProposal.go @@ -247,7 +247,7 @@ func (mp *metaProcessor) VerifyBlockProposal( } } - err = mp.checkHeaderBodyCorrelationProposal(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) + err = mp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID(), true) if err != nil { return err } diff --git a/process/block/shardblock.go b/process/block/shardblock.go index 2011589cebf..924189c4c26 100644 --- a/process/block/shardblock.go +++ b/process/block/shardblock.go @@ -145,7 +145,7 @@ func (sp *shardProcessor) ProcessBlock( go getMetricsFromBlockBody(body, sp.marshalizer, sp.appStatusHandler) - err = sp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) + err = sp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID(), false) if err != nil { return err } diff --git a/process/block/shardblockProposal.go b/process/block/shardblockProposal.go index 13d680be81b..6e9618b061f 100644 --- a/process/block/shardblockProposal.go +++ b/process/block/shardblockProposal.go @@ -179,7 +179,7 @@ func (sp *shardProcessor) VerifyBlockProposal( return process.ErrWrongTypeAssertion } - err = sp.checkHeaderBodyCorrelationProposal(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID()) + err = sp.checkHeaderBodyCorrelation(header.GetMiniBlockHeaderHandlers(), body, header.GetShardID(), true) if err != nil { return err } From 196aee5a65a2f4f9f823a48c376154cf26b0f52d Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 15:11:27 +0300 Subject: [PATCH 103/116] fixes after merge --- process/block/baseProcess.go | 8 +- process/block/baseProcess_test.go | 80 ++++++++++++++----- process/block/export_test.go | 10 +-- process/block/metablock.go | 16 ++-- process/block/metablock_test.go | 9 ++- process/block/preprocess/transactions_test.go | 2 +- 6 files changed, 84 insertions(+), 41 deletions(-) diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index 39df465dfb7..b686d55fe60 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1185,7 +1185,7 @@ func (bp *baseProcessor) checkConstructionStateProcessingTypeAndIndexesCorrectne return nil } -func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { +func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock) error { if !bytes.Equal(mbHash, mbHdr.GetHash()) { return process.ErrHeaderBodyMismatch } @@ -1220,7 +1220,7 @@ func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderWithoutConstructionAnd } func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderProposal(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { - err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock, blockShardID) + err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock) if err != nil { return err } @@ -1228,7 +1228,7 @@ func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderProposal(mbHash []byte } func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeader(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { - err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock, blockShardID) + err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock) if err != nil { return err } @@ -4136,7 +4136,7 @@ func (bp *baseProcessor) cacheUnexecutableTxHashes(headerHash []byte) { } func (bp *baseProcessor) getBlockBodyFromPool( - header data.HeaderHandler, + _ data.HeaderHandler, miniBlockHeaderHandlers []data.MiniBlockHeaderHandler, ) (data.BodyHandler, error) { miniBlocksPool := bp.dataPool.MiniBlocks() diff --git a/process/block/baseProcess_test.go b/process/block/baseProcess_test.go index 6720c67cd91..3032f7f202d 100644 --- a/process/block/baseProcess_test.go +++ b/process/block/baseProcess_test.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -21,6 +22,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/scheduled" + "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" "github.com/multiversx/mx-chain-core-go/hashing" @@ -28,9 +30,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/process/aotSelection" headersCache "github.com/multiversx/mx-chain-go/process/asyncExecution/cache" "github.com/multiversx/mx-chain-go/process/asyncExecution/executionManager" + "github.com/multiversx/mx-chain-go/testscommon/pool" "github.com/multiversx/mx-chain-go/process/asyncExecution/executionTrack" "github.com/multiversx/mx-chain-go/process/estimator" @@ -2945,23 +2949,34 @@ func TestBaseProcessor_CheckScheduledData(t *testing.T) { createProcessorAndHeader := func(t *testing.T) (interface { CheckScheduledData(data.HeaderHandler) error - }, *block.HeaderV2) { t.Helper(); coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks(); coreComponents.EnableEpochsHandlerField = enableEpochsHandlerMock.NewEnableEpochsHandlerStub(common.ScheduledMiniBlocksFlag); arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents); arguments.ArgBaseProcessor.AccountsDB[state.UserAccountsState] = &stateMock.AccountsStub{ - RootHashCalled: func() ([]byte, error) { - return []byte("scheduled-root"), nil - }, - }; arguments.ArgBaseProcessor.ScheduledTxsExecutionHandler = &testscommon.ScheduledTxsExecutionStub{ - GetScheduledGasAndFeesCalled: func() scheduled.GasAndFees { - return scheduledGasAndFees - }, - }; processor, err := blproc.NewShardProcessor(arguments); require.NoError(t, err); header := &block.HeaderV2{ - Header: &block.Header{}, - ScheduledRootHash: []byte("scheduled-root"), - ScheduledAccumulatedFees: big.NewInt(11), - ScheduledDeveloperFees: big.NewInt(12), - ScheduledGasProvided: 13, - ScheduledGasPenalized: 14, - ScheduledGasRefunded: 15, - }; return processor, header } + }, *block.HeaderV2) { + t.Helper() + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + coreComponents.EnableEpochsHandlerField = enableEpochsHandlerMock.NewEnableEpochsHandlerStub(common.ScheduledMiniBlocksFlag) + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + arguments.ArgBaseProcessor.AccountsDB[state.UserAccountsState] = &stateMock.AccountsStub{ + RootHashCalled: func() ([]byte, error) { + return []byte("scheduled-root"), nil + }, + } + arguments.ArgBaseProcessor.ScheduledTxsExecutionHandler = &testscommon.ScheduledTxsExecutionStub{ + GetScheduledGasAndFeesCalled: func() scheduled.GasAndFees { + return scheduledGasAndFees + }, + } + processor, err := blproc.NewShardProcessor(arguments) + require.NoError(t, err) + header := &block.HeaderV2{ + Header: &block.Header{}, + ScheduledRootHash: []byte("scheduled-root"), + ScheduledAccumulatedFees: big.NewInt(11), + ScheduledDeveloperFees: big.NewInt(12), + ScheduledGasProvided: 13, + ScheduledGasPenalized: 14, + ScheduledGasRefunded: 15, + } + return processor, header + } t.Run("should work when scheduled data matches", func(t *testing.T) { t.Parallel() @@ -4321,6 +4336,7 @@ func TestBaseProcessor_updateGasConsumptionLimitsIfNeeded(t *testing.T) { func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { t.Parallel() + shardID := uint32(0) t.Run("different number of miniblock headers and miniblocks should error ", func(t *testing.T) { t.Parallel() @@ -4332,6 +4348,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ {SenderShardID: 0}, }}, + shardID, ) require.Equal(t, process.ErrHeaderBodyMismatch, err) }) @@ -4348,6 +4365,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ nil, }}, + shardID, ) require.Equal(t, process.ErrNilMiniBlock, err) }) @@ -4364,6 +4382,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ {}, }}, + shardID, ) require.Equal(t, process.ErrNilMiniBlockHeader, err) }) @@ -4381,6 +4400,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ {}, }}, + shardID, ) require.Equal(t, process.ErrHeaderBodyMismatch, err) }) @@ -4404,6 +4424,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ miniBlock, }}, + shardID, ) require.Equal(t, process.ErrHeaderBodyMismatch, err) }) @@ -4429,6 +4450,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ miniBlock, }}, + shardID, ) require.ErrorIs(t, err, process.ErrHeaderBodyMismatch) }) @@ -4456,6 +4478,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ miniBlock, }}, + shardID, ) require.ErrorIs(t, err, process.ErrHeaderBodyMismatch) }) @@ -4465,8 +4488,11 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { bp, _ := blproc.NewShardProcessor(arguments) miniBlock := &block.MiniBlock{ - SenderShardID: 0, ReceiverShardID: 2, + SenderShardID: 0, + TxHashes: [][]byte{[]byte("tx1"), []byte("tx2")}, + Type: block.TxBlock, + Reserved: nil, } mbHash, _ := core.CalculateHash(arguments.CoreComponents.InternalMarshalizer(), arguments.CoreComponents.Hasher(), miniBlock) @@ -4476,6 +4502,9 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { Hash: mbHash, SenderShardID: 0, ReceiverShardID: 2, + TxCount: 2, + Type: block.TxBlock, + Reserved: nil, } _ = mbHeaders[0].SetConstructionState(int32(block.PartialExecuted)) @@ -4484,6 +4513,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ miniBlock, }}, + shardID, ) require.Equal(t, process.ErrWrongMiniBlockConstructionState, err) }) @@ -4495,6 +4525,9 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { miniBlock := &block.MiniBlock{ SenderShardID: 0, ReceiverShardID: 2, + TxHashes: [][]byte{[]byte("tx1"), []byte("tx2")}, + Type: block.TxBlock, + Reserved: nil, } mbHash, _ := core.CalculateHash(arguments.CoreComponents.InternalMarshalizer(), arguments.CoreComponents.Hasher(), miniBlock) @@ -4504,6 +4537,9 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { Hash: mbHash, SenderShardID: 0, ReceiverShardID: 2, + TxCount: 2, + Type: block.TxBlock, + Reserved: nil, } _ = mbHeaders[0].SetConstructionState(int32(block.Proposed)) _ = mbHeaders[0].SetProcessingType(int32(block.Scheduled)) @@ -4513,6 +4549,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ miniBlock, }}, + shardID, ) require.Equal(t, process.ErrWrongMiniBlockProcessingType, err) }) @@ -4524,6 +4561,9 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { miniBlock := &block.MiniBlock{ SenderShardID: 0, ReceiverShardID: 2, + TxHashes: [][]byte{[]byte("tx1"), []byte("tx2")}, + Type: block.TxBlock, + Reserved: nil, } mbHash, _ := core.CalculateHash(arguments.CoreComponents.InternalMarshalizer(), arguments.CoreComponents.Hasher(), miniBlock) @@ -4533,6 +4573,9 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { Hash: mbHash, SenderShardID: 0, ReceiverShardID: 2, + TxCount: 2, + Type: block.TxBlock, + Reserved: nil, } _ = mbHeaders[0].SetConstructionState(int32(block.Proposed)) _ = mbHeaders[0].SetProcessingType(int32(block.Normal)) @@ -4542,6 +4585,7 @@ func TestCheckHeaderBodyCorrelationProposal(t *testing.T) { &block.Body{MiniBlocks: []*block.MiniBlock{ miniBlock, }}, + shardID, ) require.NoError(t, err) }) diff --git a/process/block/export_test.go b/process/block/export_test.go index ce68fe9c49f..9c9f924442e 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -293,7 +293,7 @@ func NewShardProcessorEmptyWith3shards( }, }, BlockTracker: mock.NewBlockTrackerMock(shardCoordinator, genesisBlocks), - MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: &mock.BlockSizeThrottlerStub{}, Version: "softwareVersion", HistoryRepository: &dblookupext.HistoryRepositoryStub{}, @@ -440,7 +440,7 @@ func (mp *metaProcessor) CheckShardHeadersFinality(highestNonceHdrs map[uint32]d // CheckHeaderBodyCorrelation - func (mp *metaProcessor) CheckHeaderBodyCorrelation(hdr data.HeaderHandler, body *block.Body) error { - return mp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body, hdr.GetShardID()) + return mp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body, hdr.GetShardID(), false) } // IsHdrConstructionValid - @@ -465,7 +465,7 @@ func (sp *shardProcessor) SaveLastNotarizedHeader(shardId uint32, processedHdrs // CheckHeaderBodyCorrelation - func (sp *shardProcessor) CheckHeaderBodyCorrelation(hdr data.HeaderHandler, body *block.Body) error { - return sp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body, hdr.GetShardID()) + return sp.checkHeaderBodyCorrelation(hdr.GetMiniBlockHeaderHandlers(), body, hdr.GetShardID(), false) } // CheckAndRequestIfMetaHeadersMissing - @@ -842,8 +842,8 @@ func (bp *baseProcessor) SetMiniBlockSelectionSession(session MiniBlocksSelectio } // CheckHeaderBodyCorrelationProposal - -func (bp *baseProcessor) CheckHeaderBodyCorrelationProposal(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body) error { - return bp.checkHeaderBodyCorrelationProposal(miniBlockHeaders, body) +func (bp *baseProcessor) CheckHeaderBodyCorrelationProposal(miniBlockHeaders []data.MiniBlockHeaderHandler, body *block.Body, headerShardID uint32) error { + return bp.checkHeaderBodyCorrelation(miniBlockHeaders, body, headerShardID, true) } // GetFinalMiniBlocksFromExecutionResults - diff --git a/process/block/metablock.go b/process/block/metablock.go index 618817b7d67..862011839db 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -1940,15 +1940,13 @@ func (mp *metaProcessor) getLastSelfNotarizedHeaderByShard( } } - if lastNotarizedMetaHeader != nil { - log.Debug("last notarized meta header in shard", - "shard", shardID, - "epoch", lastNotarizedMetaHeader.GetEpoch(), - "round", lastNotarizedMetaHeader.GetRound(), - "nonce", lastNotarizedMetaHeader.GetNonce(), - "hash", lastNotarizedMetaHeaderHash, - ) - } + log.Debug("last notarized meta header in shard", + "shard", shardID, + "epoch", lastNotarizedMetaHeader.GetEpoch(), + "round", lastNotarizedMetaHeader.GetRound(), + "nonce", lastNotarizedMetaHeader.GetNonce(), + "hash", lastNotarizedMetaHeaderHash, + ) return lastNotarizedMetaHeader, lastNotarizedMetaHeaderHash } diff --git a/process/block/metablock_test.go b/process/block/metablock_test.go index 74e53404885..1b13440a5cc 100644 --- a/process/block/metablock_test.go +++ b/process/block/metablock_test.go @@ -248,7 +248,7 @@ func createMockMetaArguments( }, }, BlockTracker: blockTracker, - MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, + MiniBlockTracker: &testscommon.MiniBlockTrackerStub{}, BlockSizeThrottler: &mock.BlockSizeThrottlerStub{}, HistoryRepository: &dblookupext.HistoryRepositoryStub{}, ScheduledTxsExecutionHandler: &testscommon.ScheduledTxsExecutionStub{}, @@ -1019,16 +1019,17 @@ func TestMetaProcessor_ProcessBlock_MiniBlockChecks(t *testing.T) { coreComponents.Hash = &hashingMocks.HasherMock{} dataComponents.BlockChain = blkc bootstrapComponents.VersionedHdrFactory = &testscommon.VersionedHeaderFactoryStub{ - CreateCalled: func(epoch uint32) data.HeaderHandler { + CreateCalled: func(epoch uint32, round uint64) data.HeaderHandler { return &block.MetaBlock{ Epoch: 0, + Round: round, } }, } arguments := createMockMetaArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) arguments.TxCoordinator = txCoordinator - mp, _ := blproc.NewMetaProcessor(arguments) + mp, _ := processBlock.NewMetaProcessor(arguments) t.Run("should work with valid miniblocks", func(t *testing.T) { mb1 := &block.MiniBlock{ @@ -2527,7 +2528,7 @@ func TestMetaProcessor_SaveLastNotarizedHeader_ReleasesImmunityForCommittedShard }, } - mp, err := blproc.NewMetaProcessor(arguments) + mp, err := processBlock.NewMetaProcessor(arguments) require.Nil(t, err) const baseNonce = uint64(44) diff --git a/process/block/preprocess/transactions_test.go b/process/block/preprocess/transactions_test.go index a09ba6e2929..6e38f022606 100644 --- a/process/block/preprocess/transactions_test.go +++ b/process/block/preprocess/transactions_test.go @@ -1998,7 +1998,7 @@ func TestTransactionsPreprocessor_ProcessMiniBlockScheduledRollsBackOnError(t *t t.Run(tc.name, func(t *testing.T) { t.Parallel() args := createDefaultTransactionsProcessorArgs() - args.TxDataPool = tdp.Transactions() + args.DataPool = tdp.Transactions() txs, err := NewTransactionPreprocessor(args) require.NoError(t, err) From b7d18ad88cdd1b7363689f6de6211f31868ac8ed Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 15:52:11 +0300 Subject: [PATCH 104/116] fixes --- consensus/spos/consensusState.go | 4 ++-- integrationTests/chainSimulator/relayedTx/relayedTx_test.go | 2 +- integrationTests/multiShard/softfork/scDeploy_test.go | 1 + process/block/baseProcess.go | 2 +- .../interceptors/processor/trieNodeChunksProcessor_test.go | 2 ++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/consensus/spos/consensusState.go b/consensus/spos/consensusState.go index 3bb1611a617..f0f8e757eee 100644 --- a/consensus/spos/consensusState.go +++ b/consensus/spos/consensusState.go @@ -7,9 +7,10 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" - commonConsensus "github.com/multiversx/mx-chain-go/common/consensus" logger "github.com/multiversx/mx-chain-logger-go" + commonConsensus "github.com/multiversx/mx-chain-go/common/consensus" + "github.com/multiversx/mx-chain-go/consensus" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" @@ -75,7 +76,6 @@ func NewConsensusState( // ResetConsensusRoundState method resets all the consensus round data (except messages received) func (cns *ConsensusState) ResetConsensusRoundState() { - cns.mutState.Lock() cns.mutState.Lock() cns.roundCanceled = false cns.extendedCalled = false diff --git a/integrationTests/chainSimulator/relayedTx/relayedTx_test.go b/integrationTests/chainSimulator/relayedTx/relayedTx_test.go index 42a59c4c4c3..b6dc665d510 100644 --- a/integrationTests/chainSimulator/relayedTx/relayedTx_test.go +++ b/integrationTests/chainSimulator/relayedTx/relayedTx_test.go @@ -41,7 +41,7 @@ const ( mockTxSignature = "ssig" mockRelayerTxSignature = "rsig" maxNumOfBlocksToGenerateWhenExecutingTx = 10 - roundsPerEpoch = 40 + roundsPerEpoch = 30 guardAccountCost = 250_000 extraGasLimitForGuarded = minGasLimit extraGasESDTTransfer = 250000 diff --git a/integrationTests/multiShard/softfork/scDeploy_test.go b/integrationTests/multiShard/softfork/scDeploy_test.go index d0cdcff69a6..1374e1bb003 100644 --- a/integrationTests/multiShard/softfork/scDeploy_test.go +++ b/integrationTests/multiShard/softfork/scDeploy_test.go @@ -41,6 +41,7 @@ func TestScDeploy(t *testing.T) { enableEpochs.StakingV4Step2EnableEpoch = integrationTests.StakingV4Step2EnableEpoch enableEpochs.StakingV4Step3EnableEpoch = integrationTests.StakingV4Step3EnableEpoch enableEpochs.SupernovaEnableEpoch = integrationTests.UnreachableEpoch + enableEpochs.AndromedaEnableEpoch = integrationTests.UnreachableEpoch shardNode := integrationTests.NewTestProcessorNode(integrationTests.ArgTestProcessorNode{ MaxShards: 1, diff --git a/process/block/baseProcess.go b/process/block/baseProcess.go index b686d55fe60..e2b53abe93c 100644 --- a/process/block/baseProcess.go +++ b/process/block/baseProcess.go @@ -1219,7 +1219,7 @@ func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderWithoutConstructionAnd return nil } -func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderProposal(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, blockShardID uint32) error { +func (bp *baseProcessor) checkMiniBlockWithMiniBlockHeaderProposal(mbHash []byte, mbHdr data.MiniBlockHeaderHandler, miniBlock *block.MiniBlock, _ uint32) error { err := bp.checkMiniBlockWithMiniBlockHeaderWithoutConstructionAndProcessing(mbHash, mbHdr, miniBlock) if err != nil { return err diff --git a/process/interceptors/processor/trieNodeChunksProcessor_test.go b/process/interceptors/processor/trieNodeChunksProcessor_test.go index c9cdce1cc71..98438d595e6 100644 --- a/process/interceptors/processor/trieNodeChunksProcessor_test.go +++ b/process/interceptors/processor/trieNodeChunksProcessor_test.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data/batch" + "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process" @@ -167,6 +168,7 @@ func TestTrieNodeChunksProcessor_CheckBatchInvalidBatch(t *testing.T) { MaxChunks: 4, }, createMockWhiteLister(true), + p2p.Broadcast, ) assert.True(t, errors.Is(err, process.ErrInvalidValue)) assert.Equal(t, emptyCheckedChunkResult, chunkResult) From 41f132c6acf2db58b524f952b5acf2f062a0c049 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 16:20:02 +0300 Subject: [PATCH 105/116] fixes after merge --- consensus/spos/consensusState.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/consensus/spos/consensusState.go b/consensus/spos/consensusState.go index f0f8e757eee..5ef545ac57e 100644 --- a/consensus/spos/consensusState.go +++ b/consensus/spos/consensusState.go @@ -469,9 +469,6 @@ func (cns *ConsensusState) SetExtendedCalled(extendedCalled bool) { cns.mutState.Lock() defer cns.mutState.Unlock() - cns.mutState.Lock() - defer cns.mutState.Unlock() - cns.extendedCalled = extendedCalled } From 42892c1af5c9657cac4cfa1c4ac9e8a72b4537a0 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Mon, 8 Jun 2026 17:19:23 +0300 Subject: [PATCH 106/116] fix test --- integrationTests/consensus/consensus_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrationTests/consensus/consensus_test.go b/integrationTests/consensus/consensus_test.go index 85ad98c97bf..e6677b0e682 100644 --- a/integrationTests/consensus/consensus_test.go +++ b/integrationTests/consensus/consensus_test.go @@ -112,7 +112,7 @@ func TestConsensusBLSWithFullProcessing_WithEquivalentProofs(t *testing.T) { } enableEpochsConfig := integrationTests.CreateEnableEpochsConfig() - enableEpochsConfig.AndromedaEnableEpoch = uint32(0) + enableEpochsConfig.AndromedaEnableEpoch = uint32(1) enableEpochsConfig.SupernovaEnableEpoch = integrationTests.UnreachableEpoch numKeysOnEachNode := 1 targetEpoch := uint32(2) From 8f510f0c1edd57e63eac2ab0c3eed68a4a2b6d29 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 9 Jun 2026 10:09:42 +0300 Subject: [PATCH 107/116] fixes after review --- consensus/spos/consensusMessageValidator.go | 13 ++++++++++++- process/block/metablock.go | 16 +++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/consensus/spos/consensusMessageValidator.go b/consensus/spos/consensusMessageValidator.go index 09627073c7f..bea699d9249 100644 --- a/consensus/spos/consensusMessageValidator.go +++ b/consensus/spos/consensusMessageValidator.go @@ -544,7 +544,18 @@ func (cmv *consensusMessageValidator) removeMessageTypeToPublicKey(pk []byte, ro return } - mapMsgType[msgType]-- + count, ok := mapMsgType[msgType] + if !ok || count == 0 { + return + } + if count == 1 { + delete(mapMsgType, msgType) + if len(mapMsgType) == 0 { + delete(cmv.mapPkConsensusMessages, key) + } + return + } + mapMsgType[msgType] = count - 1 } func (cmv *consensusMessageValidator) resetConsensusMessages() { diff --git a/process/block/metablock.go b/process/block/metablock.go index 862011839db..618817b7d67 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -1940,13 +1940,15 @@ func (mp *metaProcessor) getLastSelfNotarizedHeaderByShard( } } - log.Debug("last notarized meta header in shard", - "shard", shardID, - "epoch", lastNotarizedMetaHeader.GetEpoch(), - "round", lastNotarizedMetaHeader.GetRound(), - "nonce", lastNotarizedMetaHeader.GetNonce(), - "hash", lastNotarizedMetaHeaderHash, - ) + if lastNotarizedMetaHeader != nil { + log.Debug("last notarized meta header in shard", + "shard", shardID, + "epoch", lastNotarizedMetaHeader.GetEpoch(), + "round", lastNotarizedMetaHeader.GetRound(), + "nonce", lastNotarizedMetaHeader.GetNonce(), + "hash", lastNotarizedMetaHeaderHash, + ) + } return lastNotarizedMetaHeader, lastNotarizedMetaHeaderHash } From 18d7927dd2d6f9fc1bd9b5d722d726380290d290 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 9 Jun 2026 13:23:26 +0300 Subject: [PATCH 108/116] port request SOE meta watchdog --- epochStart/shardchain/trigger.go | 102 +++++- epochStart/shardchain/triggerRegistry_test.go | 2 + epochStart/shardchain/trigger_test.go | 332 ++++++++++++++++++ process/interface.go | 1 + 4 files changed, 430 insertions(+), 7 deletions(-) diff --git a/epochStart/shardchain/trigger.go b/epochStart/shardchain/trigger.go index ebab0414fab..757eee09db2 100644 --- a/epochStart/shardchain/trigger.go +++ b/epochStart/shardchain/trigger.go @@ -19,12 +19,13 @@ import ( "github.com/multiversx/mx-chain-core-go/display" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" - logger "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("epochStart/shardchain") @@ -38,6 +39,8 @@ var _ closing.Closer = (*trigger)(nil) // sleepTime defines the time in milliseconds between each iteration made in requestMissingMiniBlocks method const sleepTime = 1 * time.Second +const numRoundsWithoutReceivedMetaBlocks = 5 + // ArgsShardEpochStartTrigger struct { defines the arguments needed for new start of epoch trigger type ArgsShardEpochStartTrigger struct { Marshalizer marshal.Marshalizer @@ -72,11 +75,12 @@ type trigger struct { epochStartShardHeader data.HeaderHandler epochStartMeta data.HeaderHandler - mutTrigger sync.RWMutex - mapHashHdr map[string]data.HeaderHandler - mapNonceHashes map[uint64][]string - mapEpochStartHdrs map[string]data.HeaderHandler - mapFinalizedEpochs map[uint32]string + mutTrigger sync.RWMutex + mapHashHdr map[string]data.HeaderHandler + mapNonceHashes map[uint64][]string + mapEpochStartHdrs map[string]data.HeaderHandler + mapFinalizedEpochs map[uint32]string + mapPreparedEpochStartHdrs map[string]struct{} headersPool dataRetriever.HeadersPool proofsPool dataRetriever.ProofsPool @@ -115,6 +119,8 @@ type trigger struct { mutMissingValidatorsInfo sync.RWMutex cancelFunc func() + chanMetaBlockReceived chan struct{} + extraDelayForRequestBlockInfo time.Duration } @@ -273,6 +279,7 @@ func NewEpochStartTrigger(args *ArgsShardEpochStartTrigger) (*trigger, error) { roundHandler: args.RoundHandler, enableEpochsHandler: args.EnableEpochsHandler, extraDelayForRequestBlockInfo: args.ExtraDelayForRequestBlockInfo, + chanMetaBlockReceived: make(chan struct{}, 1), } t.headersPool.RegisterHandler(t.receivedMetaBlock) @@ -285,11 +292,13 @@ func NewEpochStartTrigger(args *ArgsShardEpochStartTrigger) (*trigger, error) { t.mapMissingMiniBlocks = make(map[string]uint32) t.mapMissingValidatorsInfo = make(map[string]uint32) + t.mapPreparedEpochStartHdrs = make(map[string]struct{}) var ctx context.Context ctx, t.cancelFunc = context.WithCancel(context.Background()) go t.requestMissingMiniBlocks(ctx) go t.requestMissingValidatorsInfo(ctx) + go t.watchdogRequestEpochStartMetaBlock(ctx) return t, nil } @@ -333,6 +342,16 @@ func (t *trigger) requestMissingMiniBlocks(ctx context.Context) { t.mutMissingMiniBlocks.RLock() if len(t.mapMissingMiniBlocks) == 0 { t.mutMissingMiniBlocks.RUnlock() + + t.mutTrigger.Lock() + if t.isEpochStart { + t.mutTrigger.Unlock() + continue + } + + t.updateTriggerFromMeta() + t.mutTrigger.Unlock() + continue } @@ -588,6 +607,11 @@ func (t *trigger) receivedMetaBlock(headerHandler data.HeaderHandler, metaBlockH return } + select { + case t.chanMetaBlockReceived <- struct{}{}: + default: + } + log.Debug("received meta header in trigger", "header hash", metaBlockHash) if t.enableEpochsHandler.IsFlagEnabledInEpoch(common.AndromedaFlag, headerHandler.GetEpoch()) { proof, err := t.proofsPool.GetProof(headerHandler.GetShardID(), metaBlockHash) @@ -880,7 +904,10 @@ func (t *trigger) checkIfTriggerCanBeActivated(hash string, metaHdr data.HeaderH } } - t.epochStartNotifier.NotifyAllPrepare(metaHdr, blockBody) + if _, alreadyPrepared := t.mapPreparedEpochStartHdrs[hash]; !alreadyPrepared { + t.epochStartNotifier.NotifyAllPrepare(metaHdr, blockBody) + t.mapPreparedEpochStartHdrs[hash] = struct{}{} + } isMetaHdrFinal, finalityAttestingRound := t.isMetaBlockFinal(hash, metaHdr) return isMetaHdrFinal, finalityAttestingRound @@ -1095,6 +1122,7 @@ func (t *trigger) SetProcessed(header data.HeaderHandler, _ data.BodyHandler) { t.mapNonceHashes = make(map[uint64][]string) t.mapEpochStartHdrs = make(map[string]data.HeaderHandler) t.mapFinalizedEpochs = make(map[uint32]string) + t.mapPreparedEpochStartHdrs = make(map[string]struct{}) t.saveCurrentState(header.GetRound()) @@ -1253,6 +1281,66 @@ func (t *trigger) saveCurrentState(round uint64) { } } +func (t *trigger) computeWatchdogTimeout() time.Duration { + timeout := t.roundHandler.TimeDuration() * numRoundsWithoutReceivedMetaBlocks + if timeout <= 0 { + return 0 + } + return timeout +} + +func (t *trigger) watchdogRequestEpochStartMetaBlock(ctx context.Context) { + watchdogTimeout := t.computeWatchdogTimeout() + if watchdogTimeout == 0 { + return + } + + timer := time.NewTimer(watchdogTimeout) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + log.Debug("watchdogRequestEpochStartMetaBlock: trigger's go routine is stopping...") + return + case <-t.chanMetaBlockReceived: + timer.Reset(t.resetWatchdogTimeout(watchdogTimeout)) + case <-timer.C: + t.handleWatchdogTimeout() + timer.Reset(t.resetWatchdogTimeout(watchdogTimeout)) + } + } +} + +func (t *trigger) resetWatchdogTimeout(fallback time.Duration) time.Duration { + timeout := t.computeWatchdogTimeout() + if timeout == 0 { + return fallback + } + return timeout +} + +func (t *trigger) handleWatchdogTimeout() { + t.mutTrigger.RLock() + epoch := t.epoch + isEpochStart := t.isEpochStart + t.mutTrigger.RUnlock() + + if isEpochStart { + return + } + + if !t.enableEpochsHandler.IsFlagEnabledInEpoch(common.AndromedaFlag, epoch) { + return + } + + log.Debug("watchdog: no metablock received for too long, requesting epoch start metablock", + "current epoch", epoch, + "requesting epoch", epoch+1, + ) + go t.requestHandler.RequestStartOfEpochMetaBlock(epoch + 1) +} + // Close will close the endless running go routine func (t *trigger) Close() error { if t.cancelFunc != nil { diff --git a/epochStart/shardchain/triggerRegistry_test.go b/epochStart/shardchain/triggerRegistry_test.go index 970f48f6a73..68a09f9c1bf 100644 --- a/epochStart/shardchain/triggerRegistry_test.go +++ b/epochStart/shardchain/triggerRegistry_test.go @@ -57,6 +57,8 @@ func cloneTrigger(t *trigger) *trigger { rt.mapFinalizedEpochs = t.mapFinalizedEpochs rt.roundHandler = t.roundHandler rt.enableEpochsHandler = t.enableEpochsHandler + rt.chanMetaBlockReceived = t.chanMetaBlockReceived + rt.mapPreparedEpochStartHdrs = t.mapPreparedEpochStartHdrs return rt } diff --git a/epochStart/shardchain/trigger_test.go b/epochStart/shardchain/trigger_test.go index 34dd10d97be..6d6ff836243 100644 --- a/epochStart/shardchain/trigger_test.go +++ b/epochStart/shardchain/trigger_test.go @@ -1134,3 +1134,335 @@ func TestTrigger_ReceivedProof(t *testing.T) { require.True(t, wasCalled) }) } + +func TestTrigger_WatchdogRequestEpochStartMetaBlock(t *testing.T) { + t.Parallel() + + t.Run("fires after timeout", func(t *testing.T) { + t.Parallel() + + var requestedEpoch atomic.Uint32 + var called atomic.Int32 + args := createMockShardEpochStartTriggerArguments() + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 10 * time.Millisecond + }, + } + args.Epoch = 5 + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + requestedEpoch.Store(epoch) + called.Add(1) + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + time.Sleep(200 * time.Millisecond) + + require.Greater(t, called.Load(), int32(0)) + require.Equal(t, uint32(6), requestedEpoch.Load()) + }) + + t.Run("resets timer on any metablock reception", func(t *testing.T) { + t.Parallel() + + var called atomic.Int32 + args := createMockShardEpochStartTriggerArguments() + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 30 * time.Millisecond + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + called.Add(1) + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + for i := 0; i < 10; i++ { + select { + case et.chanMetaBlockReceived <- struct{}{}: + default: + } + time.Sleep(20 * time.Millisecond) + } + + require.Equal(t, int32(0), called.Load()) + }) + + t.Run("skips when epoch start already detected", func(t *testing.T) { + t.Parallel() + + var called atomic.Int32 + args := createMockShardEpochStartTriggerArguments() + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 10 * time.Millisecond + }, + IndexCalled: func() int64 { + return 100 + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + called.Add(1) + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + et.mutTrigger.Lock() + et.isEpochStart = true + et.mutTrigger.Unlock() + + time.Sleep(200 * time.Millisecond) + + require.Equal(t, int32(0), called.Load()) + }) + + t.Run("skips when Andromeda disabled", func(t *testing.T) { + t.Parallel() + + var called atomic.Int32 + args := createMockShardEpochStartTriggerArguments() + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 10 * time.Millisecond + }, + IndexCalled: func() int64 { + return 100 + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + called.Add(1) + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return false + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + time.Sleep(200 * time.Millisecond) + + require.Equal(t, int32(0), called.Load()) + }) + + t.Run("stops on context cancellation", func(t *testing.T) { + t.Parallel() + + var called atomic.Int32 + args := createMockShardEpochStartTriggerArguments() + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 10 * time.Millisecond + }, + IndexCalled: func() int64 { + return 100 + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + called.Add(1) + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + + err = et.Close() + require.Nil(t, err) + + calledBefore := called.Load() + time.Sleep(200 * time.Millisecond) + + require.Equal(t, calledBefore, called.Load()) + }) + + t.Run("does not start when TimeDuration is zero", func(t *testing.T) { + t.Parallel() + + var called atomic.Int32 + args := createMockShardEpochStartTriggerArguments() + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 0 + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + called.Add(1) + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + time.Sleep(100 * time.Millisecond) + + require.Equal(t, int32(0), called.Load()) + }) + + t.Run("receivedMetaBlock signals watchdog", func(t *testing.T) { + t.Parallel() + + args := createMockShardEpochStartTriggerArguments() + // zero TimeDuration prevents the watchdog goroutine from starting + args.RoundHandler = &mock.RoundHandlerStub{ + TimeDurationCalled: func() time.Duration { + return 0 + }, + } + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + et.receivedMetaBlock(&block.MetaBlock{ + Nonce: 10, + Round: 42, + }, []byte("hash")) + + select { + case <-et.chanMetaBlockReceived: + // expected + default: + require.Fail(t, "channel should have been signaled") + } + }) + + t.Run("receivedMetaBlock requests proof when missing in Andromeda", func(t *testing.T) { + t.Parallel() + + var proofRequested atomic.Int32 + var requestedHashMut sync.Mutex + var requestedHash []byte + args := createMockShardEpochStartTriggerArguments() + args.Epoch = 5 + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestEquivalentProofByHashCalled: func(headerShard uint32, headerHash []byte) { + requestedHashMut.Lock() + requestedHash = headerHash + requestedHashMut.Unlock() + proofRequested.Add(1) + }, + } + args.DataPool = &dataRetrieverMock.PoolsHolderStub{ + HeadersCalled: func() dataRetriever.HeadersPool { + return &mock.HeadersCacherStub{} + }, + MiniBlocksCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, + CurrEpochValidatorInfoCalled: func() dataRetriever.ValidatorInfoCacher { + return &vic.ValidatorInfoCacherStub{} + }, + ProofsCalled: func() dataRetriever.ProofsPool { + return &dataRetrieverMock.ProofsPoolMock{ + GetProofCalled: func(_ uint32, _ []byte) (data.HeaderProofHandler, error) { + return nil, errors.New("proof not found") + }, + } + }, + } + + et, err := NewEpochStartTrigger(args) + require.Nil(t, err) + defer func() { + _ = et.Close() + }() + + metaBlockHash := []byte("metablock-hash") + et.receivedMetaBlock(&block.MetaBlock{ + Nonce: 10, + Round: 42, + Epoch: 6, + EpochStart: block.EpochStart{LastFinalizedHeaders: []block.EpochStartShardData{{}}}, + }, metaBlockHash) + + time.Sleep(50 * time.Millisecond) + + require.Equal(t, int32(1), proofRequested.Load()) + requestedHashMut.Lock() + require.Equal(t, metaBlockHash, requestedHash) + requestedHashMut.Unlock() + + proofRequested.Store(0) + et.receivedMetaBlock(&block.MetaBlock{ + Nonce: 11, + Round: 43, + Epoch: 6, + }, []byte("regular-metablock-hash")) + + time.Sleep(50 * time.Millisecond) + require.Equal(t, int32(0), proofRequested.Load()) + + proofRequested.Store(0) + et.receivedMetaBlock(&block.MetaBlock{ + Nonce: 5, + Round: 30, + Epoch: 5, + EpochStart: block.EpochStart{LastFinalizedHeaders: []block.EpochStartShardData{{}}}, + }, []byte("old-epoch-start-hash")) + + time.Sleep(50 * time.Millisecond) + require.Equal(t, int32(0), proofRequested.Load()) + }) +} diff --git a/process/interface.go b/process/interface.go index e394f4c0862..ff5f924ae1a 100644 --- a/process/interface.go +++ b/process/interface.go @@ -1072,6 +1072,7 @@ type RoundTimeDurationHandler interface { // RoundHandler defines the actions which should be handled by a round implementation type RoundHandler interface { Index() int64 + TimeDuration() time.Duration IsInterfaceNil() bool } From 55c16e01a1e1d43c8df958054e1d99b803e2e441 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 9 Jun 2026 13:30:14 +0300 Subject: [PATCH 109/116] remove Andromeda request gate --- epochStart/shardchain/trigger.go | 4 ---- epochStart/shardchain/trigger_test.go | 8 ++++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/epochStart/shardchain/trigger.go b/epochStart/shardchain/trigger.go index 757eee09db2..c588a70068a 100644 --- a/epochStart/shardchain/trigger.go +++ b/epochStart/shardchain/trigger.go @@ -1330,10 +1330,6 @@ func (t *trigger) handleWatchdogTimeout() { return } - if !t.enableEpochsHandler.IsFlagEnabledInEpoch(common.AndromedaFlag, epoch) { - return - } - log.Debug("watchdog: no metablock received for too long, requesting epoch start metablock", "current epoch", epoch, "requesting epoch", epoch+1, diff --git a/epochStart/shardchain/trigger_test.go b/epochStart/shardchain/trigger_test.go index 6d6ff836243..c277d53452d 100644 --- a/epochStart/shardchain/trigger_test.go +++ b/epochStart/shardchain/trigger_test.go @@ -1251,9 +1251,10 @@ func TestTrigger_WatchdogRequestEpochStartMetaBlock(t *testing.T) { require.Equal(t, int32(0), called.Load()) }) - t.Run("skips when Andromeda disabled", func(t *testing.T) { + t.Run("fires even when Andromeda disabled", func(t *testing.T) { t.Parallel() + var requestedEpoch atomic.Uint32 var called atomic.Int32 args := createMockShardEpochStartTriggerArguments() args.RoundHandler = &mock.RoundHandlerStub{ @@ -1264,8 +1265,10 @@ func TestTrigger_WatchdogRequestEpochStartMetaBlock(t *testing.T) { return 100 }, } + args.Epoch = 5 args.RequestHandler = &testscommon.RequestHandlerStub{ RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + requestedEpoch.Store(epoch) called.Add(1) }, } @@ -1283,7 +1286,8 @@ func TestTrigger_WatchdogRequestEpochStartMetaBlock(t *testing.T) { time.Sleep(200 * time.Millisecond) - require.Equal(t, int32(0), called.Load()) + require.Greater(t, called.Load(), int32(0)) + require.Equal(t, uint32(6), requestedEpoch.Load()) }) t.Run("stops on context cancellation", func(t *testing.T) { From 682e1ff1a867a7ad0e127792854f0a1a0c61f662 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 9 Jun 2026 16:22:44 +0300 Subject: [PATCH 110/116] add meta sync watchdog for epoch start block --- factory/consensus/consensusComponents.go | 1 + integrationTests/testSyncNode.go | 1 + process/errors.go | 3 + process/sync/argBootstrapper.go | 2 + process/sync/export_test.go | 11 ++ process/sync/metablock.go | 99 ++++++++++++++ process/sync/metablock_test.go | 165 +++++++++++++++++++++++ 7 files changed, 282 insertions(+) diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index d7db14cccff..ed810e324ce 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -657,6 +657,7 @@ func (ccf *consensusComponentsFactory) createMetaChainBootstrapper() (process.Bo EpochBootstrapper: ccf.processComponents.EpochStartTrigger(), ValidatorAccountsDB: ccf.stateComponents.PeerAccounts(), ValidatorStatisticsDBSyncer: validatorAccountsDBSyncer, + Watchdog: ccf.coreComponents.Watchdog(), } return sync.NewMetaBootstrap(argsMetaBootstrapper) diff --git a/integrationTests/testSyncNode.go b/integrationTests/testSyncNode.go index 5f1a212892d..d7e3bb34665 100644 --- a/integrationTests/testSyncNode.go +++ b/integrationTests/testSyncNode.go @@ -246,6 +246,7 @@ func (tpn *TestProcessorNode) createMetaChainBootstrapper() (TestBootstrapper, e EpochBootstrapper: tpn.EpochStartTrigger, ValidatorAccountsDB: tpn.PeerState, ValidatorStatisticsDBSyncer: &mock.AccountsDBSyncerStub{}, + Watchdog: &testscommon.WatchdogMock{}, } bootstrap, err := sync.NewMetaBootstrap(argsMetaBootstrapper) diff --git a/process/errors.go b/process/errors.go index 3612db8f68c..bf627ddc706 100644 --- a/process/errors.go +++ b/process/errors.go @@ -1191,6 +1191,9 @@ var ErrNilESDTGlobalSettingsHandler = errors.New("nil esdt global settings handl // ErrNilEnableEpochsHandler signals that a nil enable epochs handler has been provided var ErrNilEnableEpochsHandler = errors.New("nil enable epochs handler") +// ErrNilWatchdog signals that a nil watchdog has been provided +var ErrNilWatchdog = errors.New("nil watchdog") + // ErrNilEpochChangeGracePeriodHandler signals that a nil epoch change grace period handler has been provided var ErrNilEpochChangeGracePeriodHandler = errors.New("nil epoch change grace period handler") diff --git a/process/sync/argBootstrapper.go b/process/sync/argBootstrapper.go index 587ecedd258..c01760c07f3 100644 --- a/process/sync/argBootstrapper.go +++ b/process/sync/argBootstrapper.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/typeConverters" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/consensus" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -65,4 +66,5 @@ type ArgMetaBootstrapper struct { EpochBootstrapper process.EpochBootstrapper ValidatorStatisticsDBSyncer process.AccountsDBSyncer ValidatorAccountsDB state.AccountsAdapter + Watchdog core.WatchdogTimer } diff --git a/process/sync/export_test.go b/process/sync/export_test.go index f8f172b733e..bb90b38659c 100644 --- a/process/sync/export_test.go +++ b/process/sync/export_test.go @@ -6,6 +6,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/process" ) @@ -35,6 +36,16 @@ func (boot *MetaBootstrap) ReceivedProof(header data.HeaderProofHandler) { boot.processReceivedProof(header) } +// RequestEpochStartBlockIfStuck - +func (boot *MetaBootstrap) RequestEpochStartBlockIfStuck() { + boot.requestEpochStartBlockIfStuck() +} + +// SetWatchdogLastNonce - +func (boot *MetaBootstrap) SetWatchdogLastNonce(nonce uint64) { + boot.watchdogLastNonce = nonce +} + // SetRcvHdrNonce - func (boot *MetaBootstrap) SetRcvHdrNonce() { boot.chRcvHdrNonce <- true diff --git a/process/sync/metablock.go b/process/sync/metablock.go index b821bfe4d3c..65a4a08922d 100644 --- a/process/sync/metablock.go +++ b/process/sync/metablock.go @@ -8,6 +8,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" + + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/state" @@ -15,12 +17,21 @@ import ( "github.com/multiversx/mx-chain-go/trie/storageMarker" ) +const ( + numRoundsWithoutCommittedBlock = 5 + metaSyncEpochStartWatchdogID = "metaSyncEpochStartWatchdog" +) + // MetaBootstrap implements the bootstrap mechanism type MetaBootstrap struct { *baseBootstrap epochBootstrapper process.EpochBootstrapper validatorStatisticsDBSyncer process.AccountsDBSyncer validatorAccountsDB state.AccountsAdapter + + watchdog core.WatchdogTimer + watchdogCtx context.Context + watchdogLastNonce uint64 } // NewMetaBootstrap creates a new Bootstrap object @@ -46,6 +57,9 @@ func NewMetaBootstrap(arguments ArgMetaBootstrapper) (*MetaBootstrap, error) { if check.IfNil(arguments.ValidatorAccountsDB) { return nil, process.ErrNilPeerAccountsAdapter } + if check.IfNil(arguments.Watchdog) { + return nil, process.ErrNilWatchdog + } err := checkBaseBootstrapParameters(arguments.ArgBaseBootstrapper) if err != nil { @@ -94,6 +108,7 @@ func NewMetaBootstrap(arguments ArgMetaBootstrapper) (*MetaBootstrap, error) { epochBootstrapper: arguments.EpochBootstrapper, validatorStatisticsDBSyncer: arguments.ValidatorStatisticsDBSyncer, validatorAccountsDB: arguments.ValidatorAccountsDB, + watchdog: arguments.Watchdog, } base.blockBootstrapper = &boot @@ -152,11 +167,91 @@ func (boot *MetaBootstrap) StartSyncingBlocks() error { var ctx context.Context ctx, boot.cancelFunc = context.WithCancel(context.Background()) + boot.watchdogCtx = ctx go boot.syncBlocks(ctx) + boot.armEpochStartWatchdog() + return nil } +func (boot *MetaBootstrap) armEpochStartWatchdog() { + if boot.watchdogCtx == nil || boot.watchdogCtx.Err() != nil { + return + } + + timeout := boot.roundHandler.TimeDuration() * numRoundsWithoutCommittedBlock + if timeout <= 0 { + return + } + + // capture the baseline nonce now so the alarm measures progress over a single interval + boot.watchdogLastNonce = boot.currentBlockNonce() + boot.watchdog.Set(boot.epochStartWatchdogCallback, timeout, metaSyncEpochStartWatchdogID) +} + +func (boot *MetaBootstrap) currentBlockNonce() uint64 { + currentHeader := boot.chainHandler.GetCurrentBlockHeader() + if check.IfNil(currentHeader) { + return 0 + } + + return currentHeader.GetNonce() +} + +func (boot *MetaBootstrap) epochStartWatchdogCallback(_ string) { + if boot.watchdogCtx == nil || boot.watchdogCtx.Err() != nil { + return + } + defer boot.armEpochStartWatchdog() + + boot.requestEpochStartBlockIfStuck() +} + +func (boot *MetaBootstrap) requestEpochStartBlockIfStuck() { + currentHeader := boot.chainHandler.GetCurrentBlockHeader() + if check.IfNil(currentHeader) { + return + } + + currentNonce := currentHeader.GetNonce() + if currentNonce != boot.watchdogLastNonce { + return + } + + currentEpoch := currentHeader.GetEpoch() + targetEpoch := currentEpoch + 1 + if !boot.enableEpochsHandler.IsFlagEnabledInEpoch(common.AndromedaFlag, targetEpoch) { + return + } + + targetNonce := currentNonce + 1 + + header, headerHash, err := process.GetMetaHeaderFromPoolWithNonce(targetNonce, boot.headers) + if err == nil && !check.IfNil(header) { + if boot.proofs.HasProof(core.MetachainShardId, headerHash) { + return + } + + log.Debug("epoch start watchdog: header present without proof, requesting proof by hash", + "nonce", targetNonce, + "epoch", header.GetEpoch(), + "hash", headerHash, + ) + boot.requestHandler.SetEpoch(header.GetEpoch()) + boot.requestHandler.RequestEquivalentProofByHash(core.MetachainShardId, headerHash) + return + } + + log.Debug("epoch start watchdog: stuck without epoch change metablock, requesting header and proof", + "nonce", targetNonce, + "epoch", targetEpoch, + ) + boot.requestHandler.SetEpoch(targetEpoch) + boot.requestHandler.RequestStartOfEpochMetaBlock(targetEpoch) + boot.requestHandler.RequestEquivalentProofByNonce(core.MetachainShardId, targetNonce) +} + func (boot *MetaBootstrap) setLastEpochStartRound() { hdr := boot.chainHandler.GetCurrentBlockHeader() if check.IfNil(hdr) || hdr.GetEpoch() < 1 { @@ -222,6 +317,10 @@ func (boot *MetaBootstrap) Close() error { return nil } + if !check.IfNil(boot.watchdog) { + boot.watchdog.Stop(metaSyncEpochStartWatchdogID) + } + return boot.baseBootstrap.Close() } diff --git a/process/sync/metablock_test.go b/process/sync/metablock_test.go index 8649b23cca1..a4438f722f3 100644 --- a/process/sync/metablock_test.go +++ b/process/sync/metablock_test.go @@ -96,11 +96,164 @@ func CreateMetaBootstrapMockArguments() sync.ArgMetaBootstrapper { EpochBootstrapper: &mock.EpochStartTriggerStub{}, ValidatorAccountsDB: &stateMock.AccountsStub{}, ValidatorStatisticsDBSyncer: &mock.AccountsDBSyncerStub{}, + Watchdog: &testscommon.WatchdogMock{}, } return argsMetaBootstrapper } +func TestMetaBootstrap_RequestEpochStartBlockIfStuck(t *testing.T) { + t.Parallel() + + const currentNonce = uint64(100) + const currentEpoch = uint32(0) + + newArgs := func() (sync.ArgMetaBootstrapper, *headerRequestsRecorder) { + recorder := &headerRequestsRecorder{} + args := CreateMetaBootstrapMockArguments() + args.ChainHandler = &testscommon.ChainHandlerStub{ + GetCurrentBlockHeaderCalled: func() data.HeaderHandler { + return &block.MetaBlock{Nonce: currentNonce, Epoch: currentEpoch} + }, + } + args.RequestHandler = &testscommon.RequestHandlerStub{ + RequestStartOfEpochMetaBlockCalled: func(epoch uint32) { + recorder.startOfEpochCalls++ + recorder.startOfEpochArg = epoch + }, + RequestEquivalentProofByHashCalled: func(headerShard uint32, headerHash []byte) { + recorder.proofByHashCalls++ + recorder.proofByHashArg = headerHash + }, + RequestEquivalentProofByNonceCalled: func(headerShard uint32, headerNonce uint64) { + recorder.proofByNonceCalls++ + recorder.proofByNonceArg = headerNonce + }, + } + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return flag == common.AndromedaFlag + }, + } + return args, recorder + } + + t.Run("not stuck (nonce advanced since last arm) does not request anything", func(t *testing.T) { + t.Parallel() + + args, recorder := newArgs() + boot, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) + + // baseline is behind the current nonce -> progress was made during the interval + boot.SetWatchdogLastNonce(currentNonce - 1) + boot.RequestEpochStartBlockIfStuck() + require.Zero(t, recorder.total()) + }) + + t.Run("stuck, header absent, requests start of epoch block and proof by nonce", func(t *testing.T) { + t.Parallel() + + args, recorder := newArgs() + boot, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) + + boot.SetWatchdogLastNonce(currentNonce) + boot.RequestEpochStartBlockIfStuck() + + require.Equal(t, 1, recorder.startOfEpochCalls) + require.Equal(t, currentEpoch+1, recorder.startOfEpochArg) + require.Equal(t, 1, recorder.proofByNonceCalls) + require.Equal(t, currentNonce+1, recorder.proofByNonceArg) + require.Equal(t, 0, recorder.proofByHashCalls) + }) + + t.Run("stuck, header present without proof, requests proof by hash", func(t *testing.T) { + t.Parallel() + + expectedHash := []byte("epoch-change-hash") + args, recorder := newArgs() + args.PoolsHolder = poolsWithMetaHeader(currentNonce+1, currentEpoch+1, expectedHash, false) + boot, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) + + boot.SetWatchdogLastNonce(currentNonce) + boot.RequestEpochStartBlockIfStuck() + + require.Equal(t, 1, recorder.proofByHashCalls) + require.Equal(t, expectedHash, recorder.proofByHashArg) + require.Equal(t, 0, recorder.startOfEpochCalls) + require.Equal(t, 0, recorder.proofByNonceCalls) + }) + + t.Run("stuck, header present with proof, does not request anything", func(t *testing.T) { + t.Parallel() + + args, recorder := newArgs() + args.PoolsHolder = poolsWithMetaHeader(currentNonce+1, currentEpoch+1, []byte("hash"), true) + boot, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) + + boot.SetWatchdogLastNonce(currentNonce) + boot.RequestEpochStartBlockIfStuck() + + require.Zero(t, recorder.total()) + }) + + t.Run("stuck but andromeda not enabled does not request anything", func(t *testing.T) { + t.Parallel() + + args, recorder := newArgs() + args.EnableEpochsHandler = &enableEpochsHandlerMock.EnableEpochsHandlerStub{ + IsFlagEnabledInEpochCalled: func(flag core.EnableEpochFlag, epoch uint32) bool { + return false + }, + } + boot, err := sync.NewMetaBootstrap(args) + require.Nil(t, err) + + boot.SetWatchdogLastNonce(currentNonce) + boot.RequestEpochStartBlockIfStuck() + + require.Zero(t, recorder.total()) + }) +} + +type headerRequestsRecorder struct { + startOfEpochCalls int + startOfEpochArg uint32 + proofByHashCalls int + proofByHashArg []byte + proofByNonceCalls int + proofByNonceArg uint64 +} + +func (r *headerRequestsRecorder) total() int { + return r.startOfEpochCalls + r.proofByHashCalls + r.proofByNonceCalls +} + +func poolsWithMetaHeader(nonce uint64, epoch uint32, hash []byte, hasProof bool) *dataRetrieverMock.PoolsHolderStub { + pools := createMockPools() + pools.HeadersCalled = func() dataRetriever.HeadersPool { + return &mock.HeadersCacherStub{ + GetHeaderByNonceAndShardIdCalled: func(hdrNonce uint64, shardId uint32) ([]data.HeaderHandler, [][]byte, error) { + if hdrNonce != nonce { + return nil, nil, errors.New("not found") + } + return []data.HeaderHandler{&block.MetaBlock{Nonce: nonce, Epoch: epoch}}, [][]byte{hash}, nil + }, + } + } + pools.ProofsCalled = func() dataRetriever.ProofsPool { + return &dataRetrieverMock.ProofsPoolMock{ + HasProofCalled: func(shardID uint32, headerHash []byte) bool { + return hasProof + }, + } + } + return pools +} + // ------- NewMetaBootstrap func TestNewMetaBootstrap_NilPoolsHolderShouldErr(t *testing.T) { @@ -115,6 +268,18 @@ func TestNewMetaBootstrap_NilPoolsHolderShouldErr(t *testing.T) { assert.Equal(t, process.ErrNilPoolsHolder, err) } +func TestNewMetaBootstrap_NilWatchdogShouldErr(t *testing.T) { + t.Parallel() + + args := CreateMetaBootstrapMockArguments() + args.Watchdog = nil + + bs, err := sync.NewMetaBootstrap(args) + + assert.True(t, check.IfNil(bs)) + assert.Equal(t, process.ErrNilWatchdog, err) +} + func TestNewMetaBootstrap_NilValidatorDBShouldErr(t *testing.T) { t.Parallel() From 98c228b964c5a3763ce48cbbbeb24f3b9963b9b8 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Tue, 9 Jun 2026 17:14:23 +0300 Subject: [PATCH 111/116] fix after review --- epochStart/bootstrap/process.go | 9 +++++++-- epochStart/bootstrap/storageProcess.go | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 13469fe333b..1fa6be7ce49 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -1582,9 +1582,14 @@ func (e *epochStartBootstrap) createResolversContainer() error { return err } - e.resolversContainer = container + err = resolverFactory.AddShardTrieNodeResolvers(container) + if err != nil { + _ = container.Close() + return err + } - return resolverFactory.AddShardTrieNodeResolvers(container) + e.resolversContainer = container + return nil } func (e *epochStartBootstrap) createRequestHandler() error { diff --git a/epochStart/bootstrap/storageProcess.go b/epochStart/bootstrap/storageProcess.go index 34dbfecbe73..ce88eddf679 100644 --- a/epochStart/bootstrap/storageProcess.go +++ b/epochStart/bootstrap/storageProcess.go @@ -318,24 +318,32 @@ func (sesb *storageEpochStartBootstrap) rebuildStorageComponentsForShard() error } func (sesb *storageEpochStartBootstrap) closeStorageRequesters() error { - var errFound error + var containerErr error if !check.IfNil(sesb.container) { err := sesb.container.Close() if err != nil { - errFound = fmt.Errorf("close storage requesters container: %w", err) + containerErr = fmt.Errorf("close storage requesters container: %w", err) } sesb.container = nil } + var storeErr error if !check.IfNil(sesb.store) { err := sesb.store.CloseAll() if err != nil { - errFound = fmt.Errorf("close storage service: %w", err) + storeErr = fmt.Errorf("close storage service: %w", err) } sesb.store = nil } - return errFound + switch { + case containerErr != nil && storeErr != nil: + return fmt.Errorf("%v; %w", containerErr, storeErr) + case storeErr != nil: + return storeErr + default: + return containerErr + } } func (sesb *storageEpochStartBootstrap) requestAndProcessFromStorage() (Parameters, error) { From 2064a96fbc7f6e9183e1a379221af0c76edfa9e1 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 10 Jun 2026 11:18:44 +0300 Subject: [PATCH 112/116] fix after review --- epochStart/shardchain/trigger.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/epochStart/shardchain/trigger.go b/epochStart/shardchain/trigger.go index c588a70068a..471842f04a3 100644 --- a/epochStart/shardchain/trigger.go +++ b/epochStart/shardchain/trigger.go @@ -1298,16 +1298,26 @@ func (t *trigger) watchdogRequestEpochStartMetaBlock(ctx context.Context) { timer := time.NewTimer(watchdogTimeout) defer timer.Stop() + resetTimer := func(d time.Duration) { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(d) + } + for { select { case <-ctx.Done(): log.Debug("watchdogRequestEpochStartMetaBlock: trigger's go routine is stopping...") return case <-t.chanMetaBlockReceived: - timer.Reset(t.resetWatchdogTimeout(watchdogTimeout)) + resetTimer(t.resetWatchdogTimeout(watchdogTimeout)) case <-timer.C: t.handleWatchdogTimeout() - timer.Reset(t.resetWatchdogTimeout(watchdogTimeout)) + resetTimer(t.resetWatchdogTimeout(watchdogTimeout)) } } } From 1dfe15095866cfdb829fbcf5e1ee39b136d1c4d5 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 10 Jun 2026 14:02:20 +0300 Subject: [PATCH 113/116] fix tests --- .../chainParametersNotifier_test.go | 8 +++-- epochStart/shardchain/triggerRegistry_test.go | 33 +++++++++++++++---- .../staking/stake/stakeAndUnStake_test.go | 28 ++++++++-------- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/common/chainparametersnotifier/chainParametersNotifier_test.go b/common/chainparametersnotifier/chainParametersNotifier_test.go index fa1a30959d4..9bca337249c 100644 --- a/common/chainparametersnotifier/chainParametersNotifier_test.go +++ b/common/chainparametersnotifier/chainParametersNotifier_test.go @@ -5,8 +5,9 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-go/config" "github.com/stretchr/testify/require" + + "github.com/multiversx/mx-chain-go/config" ) func TestNewChainParametersNotifier(t *testing.T) { @@ -112,12 +113,15 @@ func TestChainParametersNotifier_ConcurrentOperations(t *testing.T) { } type dummyNotifee struct { - receivedChainParameters config.ChainParametersByEpochConfig + receivedChainParameters config.ChainParametersByEpochConfig + mutReceivedChainParameters sync.RWMutex } // ChainParametersChanged - func (dn *dummyNotifee) ChainParametersChanged(chainParameters config.ChainParametersByEpochConfig) { + dn.mutReceivedChainParameters.Lock() dn.receivedChainParameters = chainParameters + dn.mutReceivedChainParameters.Unlock() } // IsInterfaceNil - diff --git a/epochStart/shardchain/triggerRegistry_test.go b/epochStart/shardchain/triggerRegistry_test.go index 68a09f9c1bf..3d2d42803a8 100644 --- a/epochStart/shardchain/triggerRegistry_test.go +++ b/epochStart/shardchain/triggerRegistry_test.go @@ -18,6 +18,8 @@ import ( func cloneTrigger(t *trigger) *trigger { rt := &trigger{} + t.mutTrigger.RLock() + defer t.mutTrigger.RUnlock() rt.epoch = t.epoch rt.metaEpoch = t.epoch @@ -67,6 +69,9 @@ func createDummyEpochStartTriggers(arguments *ArgsShardEpochStartTrigger, key [] // create a copy epochStartTrigger2 := cloneTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.RLock() + defer epochStartTrigger1.mutTrigger.RUnlock() + epochStartTrigger1.triggerStateKey = key epochStartTrigger1.epoch = 10 epochStartTrigger1.metaEpoch = 11 @@ -95,13 +100,19 @@ func TestTrigger_LoadHeaderV1StateAfterSave(t *testing.T) { } key := []byte("key") epochStartTrigger1, epochStartTrigger2 := createDummyEpochStartTriggers(arguments, key) + + epochStartTrigger1.mutTrigger.RLock() err := epochStartTrigger1.saveState(key) + epochStartTrigger1.mutTrigger.RUnlock() + assert.Nil(t, err) - assert.NotEqual(t, epochStartTrigger1, epochStartTrigger2) + trigger1Clone := cloneTrigger(epochStartTrigger1) + assert.NotEqual(t, trigger1Clone, epochStartTrigger2) err = epochStartTrigger2.LoadState(key) assert.Nil(t, err) - assert.Equal(t, epochStartTrigger1, epochStartTrigger2) + trigger2Clone := cloneTrigger(epochStartTrigger2) + assert.Equal(t, trigger1Clone, trigger2Clone) } func TestTrigger_LoadHeaderV2StateAfterSave(t *testing.T) { @@ -123,13 +134,18 @@ func TestTrigger_LoadHeaderV2StateAfterSave(t *testing.T) { epochStartTrigger1.epochStartShardHeader = &block.HeaderV2{ Header: &block.Header{}, ScheduledRootHash: []byte("scheduled root hash")} + + epochStartTrigger1.mutTrigger.RLock() err := epochStartTrigger1.saveState(key) + epochStartTrigger1.mutTrigger.RUnlock() assert.Nil(t, err) - assert.NotEqual(t, epochStartTrigger1, epochStartTrigger2) + trigger1Clone := cloneTrigger(epochStartTrigger1) + assert.NotEqual(t, trigger1Clone, epochStartTrigger2) err = epochStartTrigger2.LoadState(key) assert.Nil(t, err) - assert.Equal(t, epochStartTrigger1, epochStartTrigger2) + trigger2Clone := cloneTrigger(epochStartTrigger2) + assert.Equal(t, trigger1Clone, trigger2Clone) } func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { @@ -149,7 +165,12 @@ func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { key := []byte("key") epochStartTrigger1, epochStartTrigger2 := createDummyEpochStartTriggers(arguments, key) - trig := createLegacyTriggerRegistryFromTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.RLock() + trigger1Clone := cloneTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.RUnlock() + + trig := createLegacyTriggerRegistryFromTrigger(trigger1Clone) + d, err := json.Marshal(trig) require.Nil(t, err) trigInternalKey := append([]byte(common.TriggerRegistryKeyPrefix), key...) @@ -159,7 +180,7 @@ func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { err = epochStartTrigger2.LoadState(key) require.Nil(t, err) - require.Equal(t, epochStartTrigger1, epochStartTrigger2) + require.Equal(t, trigger1Clone, epochStartTrigger2) } type legacyTriggerRegistry struct { diff --git a/integrationTests/chainSimulator/staking/stake/stakeAndUnStake_test.go b/integrationTests/chainSimulator/staking/stake/stakeAndUnStake_test.go index 471ff3ec1a7..f31649181df 100644 --- a/integrationTests/chainSimulator/staking/stake/stakeAndUnStake_test.go +++ b/integrationTests/chainSimulator/staking/stake/stakeAndUnStake_test.go @@ -49,7 +49,7 @@ func TestChainSimulator_AddValidatorKey(t *testing.T) { } startTime := time.Now().Unix() - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 20, @@ -185,7 +185,7 @@ func TestChainSimulator_AddANewValidatorAfterStakingV4(t *testing.T) { } startTime := time.Now().Unix() - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 20, @@ -216,7 +216,7 @@ func TestChainSimulator_AddANewValidatorAfterStakingV4(t *testing.T) { defer cs.Close() - err = cs.GenerateBlocks(150) + err = cs.GenerateBlocks(100) require.Nil(t, err) // Step 1 --- add a new validator key in the chain simulator @@ -280,7 +280,7 @@ func TestChainSimulator_AddANewValidatorAfterStakingV4(t *testing.T) { require.Equal(t, 20, len(results[0].Nodes)) checkTotalQualified(t, results, 8) - err = cs.GenerateBlocks(100) + err = cs.GenerateBlocks(60) require.Nil(t, err) results, err = cs.GetNodeHandler(core.MetachainShardId).GetFacadeHandler().AuctionListApi() @@ -317,7 +317,7 @@ func TestChainSimulatorStakeUnStakeUnBond(t *testing.T) { func testStakeUnStakeUnBond(t *testing.T, targetEpoch int32) { startTime := time.Now().Unix() - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -447,7 +447,7 @@ func TestChainSimulator_DirectStakingNodes_StakeFunds(t *testing.T) { t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -666,7 +666,7 @@ func TestChainSimulator_DirectStakingNodes_UnstakeFundsWithDeactivation(t *testi t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -948,7 +948,7 @@ func TestChainSimulator_DirectStakingNodes_UnstakeFundsWithDeactivation_WithReac t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -1192,7 +1192,7 @@ func TestChainSimulator_DirectStakingNodes_WithdrawUnstakedFundsBeforeUnbonding( t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -1428,7 +1428,7 @@ func TestChainSimulator_DirectStakingNodes_WithdrawUnstakedInWithdrawEpoch(t *te t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -1693,7 +1693,7 @@ func TestChainSimulator_DirectStakingNodes_WithdrawUnstakedInBatches(t *testing. t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -2060,7 +2060,7 @@ func TestChainSimulator_DirectStakingNodes_WithdrawUnstakedInEpoch(t *testing.T) t.Skip("this is not a short test") } - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -2358,7 +2358,7 @@ func TestChainSimulator_UnStakeOneActiveNodeAndCheckAPIAuctionList(t *testing.T) } startTime := time.Now().Unix() - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 30, @@ -2437,7 +2437,7 @@ func TestChainSimulator_EdgeCaseLowWaitingList(t *testing.T) { } startTime := time.Now().Unix() - roundDurationInMillis := uint64(6000) + roundDurationInMillis := uint64(1000) roundsPerEpoch := core.OptionalUint64{ HasValue: true, Value: 20, From 48c6755fd49ae4bc245ffada038a8c8921dcc140 Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 10 Jun 2026 14:16:57 +0300 Subject: [PATCH 114/116] fix lock --- epochStart/shardchain/triggerRegistry_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/epochStart/shardchain/triggerRegistry_test.go b/epochStart/shardchain/triggerRegistry_test.go index 3d2d42803a8..dce6cefef22 100644 --- a/epochStart/shardchain/triggerRegistry_test.go +++ b/epochStart/shardchain/triggerRegistry_test.go @@ -69,8 +69,8 @@ func createDummyEpochStartTriggers(arguments *ArgsShardEpochStartTrigger, key [] // create a copy epochStartTrigger2 := cloneTrigger(epochStartTrigger1) - epochStartTrigger1.mutTrigger.RLock() - defer epochStartTrigger1.mutTrigger.RUnlock() + epochStartTrigger1.mutTrigger.Lock() + defer epochStartTrigger1.mutTrigger.Unlock() epochStartTrigger1.triggerStateKey = key epochStartTrigger1.epoch = 10 From e3af0a4d21758da0a59b964ab24b87f61865c1bc Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Wed, 10 Jun 2026 15:36:23 +0300 Subject: [PATCH 115/116] fixes after merge --- epochStart/bootstrap/process.go | 34 +++++++++---------- epochStart/bootstrap/process_test.go | 8 +++-- epochStart/shardchain/triggerRegistry_test.go | 21 ++++++++---- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 271dc39e8f0..cba506bc09d 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -1921,24 +1921,22 @@ func (e *epochStartBootstrap) createResolversContainer() error { log.Debug("epochStartBootstrap.createResolversContainer", "shard", e.shardCoordinator.SelfId()) resolversContainerArgs := resolverscontainer.FactoryArgs{ - ShardCoordinator: e.shardCoordinator, - MainMessenger: e.mainMessenger, - FullArchiveMessenger: e.fullArchiveMessenger, - Store: storageService, - Marshalizer: e.coreComponentsHolder.InternalMarshalizer(), - DataPools: e.dataPool, - Uint64ByteSliceConverter: uint64ByteSlice.NewBigEndianConverter(), - NumConcurrentResolvingJobs: 10, - NumConcurrentResolvingTrieNodesJobs: 3, - DataPacker: dataPacker, - TriesContainer: e.trieContainer, - SizeCheckDelta: 0, - InputAntifloodHandler: disabled.NewAntiFloodHandler(), - OutputAntifloodHandler: disabled.NewAntiFloodHandler(), - MainPreferredPeersHolder: disabled.NewPreferredPeersHolder(), - FullArchivePreferredPeersHolder: disabled.NewPreferredPeersHolder(), - PayloadValidator: payloadValidator, - AntifloodConfigsHandler: e.coreComponentsHolder.AntifloodConfigsHandler(), + ShardCoordinator: e.shardCoordinator, + MainMessenger: e.mainMessenger, + FullArchiveMessenger: e.fullArchiveMessenger, + Store: storageService, + Marshalizer: e.coreComponentsHolder.InternalMarshalizer(), + DataPools: e.dataPool, + Uint64ByteSliceConverter: uint64ByteSlice.NewBigEndianConverter(), + DataPacker: dataPacker, + TriesContainer: e.trieContainer, + SizeCheckDelta: 0, + InputAntifloodHandler: disabled.NewAntiFloodHandler(), + OutputAntifloodHandler: disabled.NewAntiFloodHandler(), + MainPreferredPeersHolder: disabled.NewPreferredPeersHolder(), + FullArchivePreferredPeersHolder: disabled.NewPreferredPeersHolder(), + PayloadValidator: payloadValidator, + AntifloodConfigsHandler: e.coreComponentsHolder.AntifloodConfigsHandler(), } var resolverFactory dataRetriever.ResolversContainerFactory if e.shardCoordinator.SelfId() == core.MetachainShardId { diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 1ada50aac3d..30c4feb54f9 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -19,6 +19,9 @@ import ( dataBatch "github.com/multiversx/mx-chain-core-go/data/batch" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/graceperiod" "github.com/multiversx/mx-chain-go/common/statistics" @@ -60,8 +63,6 @@ import ( validatorInfoCacherStub "github.com/multiversx/mx-chain-go/testscommon/validatorInfoCacher" "github.com/multiversx/mx-chain-go/trie/factory" updateMock "github.com/multiversx/mx-chain-go/update/mock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) var errExpected = errors.New("expected error") @@ -1238,6 +1239,9 @@ func buildRebuildTestDataPool() dataRetriever.PoolsHolder { ProofsCalled: func() dataRetriever.ProofsPool { return &dataRetrieverMock.ProofsPoolMock{} }, + DirectSentTransactionsCalled: func() storage.Cacher { + return cache.NewCacherStub() + }, } } diff --git a/epochStart/shardchain/triggerRegistry_test.go b/epochStart/shardchain/triggerRegistry_test.go index b293724545f..dd3e4730c68 100644 --- a/epochStart/shardchain/triggerRegistry_test.go +++ b/epochStart/shardchain/triggerRegistry_test.go @@ -169,10 +169,10 @@ func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { epochStartTrigger1, epochStartTrigger2 := createDummyEpochStartTriggers(arguments, key) epochStartTrigger1.mutTrigger.RLock() - trigger1Clone := cloneTrigger(epochStartTrigger1) - epochStartTrigger1.mutTrigger.RUnlock() + trigger1Clone := cloneTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.RUnlock() - trig := createLegacyTriggerRegistryFromTrigger(trigger1Clone) + trig := createLegacyTriggerRegistryFromTrigger(trigger1Clone) d, _ := json.Marshal(trig) trigInternalKey := append([]byte(common.TriggerRegistryKeyPrefix), key...) @@ -208,13 +208,16 @@ func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { require.Nil(t, err) epochStartTrigger2 := cloneTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.Lock() epochStartTrigger1.epoch = epoch epochStartTrigger1.triggerStateKey = key epochStartTrigger1.cancelFunc = nil + epochStartTrigger1.mutTrigger.Unlock() err = epochStartTrigger2.LoadState(key) require.Nil(t, err) - require.Equal(t, epochStartTrigger1, epochStartTrigger2) + triggerClone := cloneTrigger(epochStartTrigger1) + require.Equal(t, triggerClone, epochStartTrigger2) }) t.Run("header v2", func(t *testing.T) { @@ -243,16 +246,19 @@ func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { require.Nil(t, err) epochStartTrigger2 := cloneTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.Lock() epochStartTrigger1.epoch = epoch epochStartTrigger1.triggerStateKey = key epochStartTrigger1.epochStartShardHeader = &block.HeaderV2{ Header: &block.Header{}, } epochStartTrigger1.cancelFunc = nil + epochStartTrigger1.mutTrigger.Unlock() err = epochStartTrigger2.LoadState(key) require.Nil(t, err) - require.Equal(t, epochStartTrigger1, epochStartTrigger2) + triggerClone := cloneTrigger(epochStartTrigger1) + require.Equal(t, triggerClone, epochStartTrigger2) }) t.Run("header v3", func(t *testing.T) { @@ -277,14 +283,17 @@ func TestTrigger_LoadStateBackwardsCompatibility(t *testing.T) { require.Nil(t, err) epochStartTrigger2 := cloneTrigger(epochStartTrigger1) + epochStartTrigger1.mutTrigger.Lock() epochStartTrigger1.epoch = epoch epochStartTrigger1.triggerStateKey = key epochStartTrigger1.epochStartShardHeader = &block.HeaderV3{} epochStartTrigger1.cancelFunc = nil + epochStartTrigger1.mutTrigger.Unlock() err = epochStartTrigger2.LoadState(key) require.Nil(t, err) - require.Equal(t, trigger1Clone, epochStartTrigger2) + triggerClone := cloneTrigger(epochStartTrigger1) + require.Equal(t, triggerClone, epochStartTrigger2) }) } From fc64c23c839fd742b5772335c99306b816b3181a Mon Sep 17 00:00:00 2001 From: Adrian Dobrita Date: Thu, 11 Jun 2026 10:42:43 +0300 Subject: [PATCH 116/116] fix bootstrap requests after merge --- epochStart/bootstrap/process.go | 5 ++ epochStart/bootstrap/process_test.go | 83 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index cba506bc09d..6e675c8ffa3 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -711,7 +711,12 @@ func (e *epochStartBootstrap) syncHeadersV3From(meta data.MetaHeaderHandler) (ma hashesToRequest := make([][]byte, 0) shardIds := make([]uint32, 0) + isCurrentShardMeta := e.shardCoordinator.SelfId() == core.MetachainShardId for _, epochStartData := range meta.GetEpochStartHandler().GetLastFinalizedHeaderHandlers() { + if !isCurrentShardMeta && epochStartData.GetShardID() != e.shardCoordinator.SelfId() { + continue + } + err := e.syncEpochStartDataInfo(meta, epochStartData, syncedHeaders) if err != nil { return nil, err diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 30c4feb54f9..e6d6f0942d3 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -3551,6 +3551,89 @@ func TestEpochStartBoostrap_SyncHeadersV3FromMeta(t *testing.T) { require.Nil(t, headers) }) + t.Run("shard node should not request other shards epoch start data", func(t *testing.T) { + t.Parallel() + + hdrHash1 := []byte("hdrHash1") + hdrHash2 := []byte("hdrHash2") + otherShardHdrHash := []byte("otherShardHdrHash") + lastExecMetaHash := []byte("lastExecMetaHash") + + header1 := &block.Header{ + Nonce: 11, + PrevHash: hdrHash2, + } + + lastExecMeta := &block.MetaBlockV3{ + Nonce: 20, + LastExecutionResult: &block.MetaExecutionResultInfo{ + ExecutionResult: &block.BaseMetaExecutionResult{ + BaseExecutionResult: &block.BaseExecutionResult{}, + }, + }, + } + + coreComp, cryptoComp := createComponentsForEpochStart() + args := createMockEpochStartBootstrapArgs(coreComp, cryptoComp) + + epochStartProvider, _ := NewEpochStartBootstrap(args) + require.Equal(t, uint32(0), epochStartProvider.shardCoordinator.SelfId()) + + epochStartProvider.headersSyncer = &epochStartMocks.HeadersByHashSyncerStub{ + SyncMissingHeadersByHashCalled: func(shardIDs []uint32, headersHashes [][]byte, ctx context.Context) error { + for _, hash := range headersHashes { + require.NotEqual(t, otherShardHdrHash, hash) + } + for _, shardID := range shardIDs { + require.True(t, shardID == 0 || shardID == core.MetachainShardId) + } + return nil + }, + GetHeadersCalled: func() (m map[string]data.HeaderHandler, err error) { + return map[string]data.HeaderHandler{ + string(hdrHash1): header1, + string(lastExecMetaHash): lastExecMeta, + }, nil + }, + } + + metaBlock := &block.MetaBlockV3{ + Epoch: 2, + Nonce: 21, + PrevHash: lastExecMetaHash, + EpochStart: block.EpochStart{ + LastFinalizedHeaders: []block.EpochStartShardData{ + { + HeaderHash: hdrHash1, + ShardID: 0, + LastFinishedMetaBlock: lastExecMetaHash, + }, + { + HeaderHash: otherShardHdrHash, + ShardID: 1, + LastFinishedMetaBlock: lastExecMetaHash, + }, + }, + Economics: block.Economics{ + PrevEpochStartHash: hdrHash2, + }, + }, + LastExecutionResult: &block.MetaExecutionResultInfo{ + ExecutionResult: &block.BaseMetaExecutionResult{ + BaseExecutionResult: &block.BaseExecutionResult{ + HeaderNonce: 20, + HeaderHash: lastExecMetaHash, + }, + }, + }, + } + + headers, err := epochStartProvider.syncHeadersFrom(metaBlock) + require.Nil(t, err) + require.Equal(t, 2, len(headers)) + require.NotContains(t, headers, string(otherShardHdrHash)) + }) + t.Run("should work with meta v3 and shard v2", func(t *testing.T) { t.Parallel()