meta finality view and settlement checker integration

factory/consensus/consensusComponents.go

@@ -28,6 +28,7 @@ import (
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/process/sync"
 	"github.com/multiversx/mx-chain-go/process/sync/storageBootstrap"
+	"github.com/multiversx/mx-chain-go/process/track"
 	"github.com/multiversx/mx-chain-go/sharding"
 	nodesCoord "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
 	"github.com/multiversx/mx-chain-go/state/syncer"
@@ -535,8 +536,18 @@ func (ccf *consensusComponentsFactory) createShardBootstrapper() (process.Bootst
 		ProcessConfigsHandler:        ccf.coreComponents.ProcessConfigsHandler(),
 	}
 
+	dataPool := ccf.dataComponents.Datapool()
+	metaFinalityView, err := track.NewMetaFinalityView(track.ArgsMetaFinalityView{
+		HeadersPool: dataPool.Headers(),
+		ProofsPool:  dataPool.Proofs(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	argsShardBootstrapper := sync.ArgShardBootstrapper{
 		ArgBaseBootstrapper: argsBaseBootstrapper,
+		MetaFinalityView:    metaFinalityView,
 	}
 
 	return sync.NewShardBootstrap(argsShardBootstrapper)

integrationTests/sync/basicSync/supernovaReconcile_test.go

@@ -10,8 +10,30 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/multiversx/mx-chain-go/integrationTests"
+	"github.com/multiversx/mx-chain-go/process"
 )
 
+// mirrors process/block.metaArbitrationWindowRounds, the R-RESOLVE discovery window
+const metaArbitrationWindowRounds = 3
+
+func requireNotarizes(t *testing.T, metaHeader data.HeaderHandler, shardID uint32, hashes ...[]byte) {
+	metaHandler, ok := metaHeader.(data.MetaHeaderHandler)
+	require.True(t, ok)
+
+	for _, shardInfo := range process.GetShardHeadersReferencedByMeta(metaHandler) {
+		if shardInfo.GetShardID() != shardID {
+			continue
+		}
+		for _, hash := range hashes {
+			if string(shardInfo.GetHeaderHash()) == string(hash) {
+				return
+			}
+		}
+	}
+
+	require.Fail(t, "meta block does not notarize the winning branch")
+}
+
 // backstop scenario: island 1 instantly finalizes clean sibling A; island 2,
 // blind to A, commits contended sibling B AND extends it with a proofed child C.
 // When B and C reach island 1, its nodes hold a FINALIZED block that objectively
@@ -55,12 +77,14 @@ func TestSupernovaSync_ReconcileBackstop_FinalizedMinorityConverges(t *testing.T
 		RoundsConfig:         &roundsConfig,
 	})
 
-	island1 := []*integrationTests.TestProcessorNode{pA, obsA, metaNode}
-	island2 := []*integrationTests.TestProcessorNode{pB, obsB}
+	// meta sits on the winning side: it never learns A, so it arbitrates B and its notarization is
+	// the authority verdict delivered to the stranded island
+	island1 := []*integrationTests.TestProcessorNode{pA, obsA}
+	island2 := []*integrationTests.TestProcessorNode{pB, obsB, metaNode}
 	allNodes := []*integrationTests.TestProcessorNode{pA, obsA, metaNode, pB, obsB}
 
-	integrationTests.ConnectNodes([]integrationTests.Connectable{pA, obsA, metaNode})
-	integrationTests.ConnectNodes([]integrationTests.Connectable{pB, obsB})
+	integrationTests.ConnectNodes([]integrationTests.Connectable{pA, obsA})
+	integrationTests.ConnectNodes([]integrationTests.Connectable{pB, obsB, metaNode})
 
 	defer func() {
 		for _, n := range allNodes {
@@ -100,14 +124,22 @@ func TestSupernovaSync_ReconcileBackstop_FinalizedMinorityConverges(t *testing.T
 	integrationTests.UpdateRound(allNodes, round)
 	nonce++
 
-	// common prefix, rounds 1..4
+	// common prefix, rounds 1..4: the shard proposes on island 1, meta notarizes it on island 2,
+	// and each side is fed the other's block
 	numPrefixBlocks := 4
 	for i := 0; i < numPrefixBlocks; i++ {
-		integrationTests.ProposeBlockWithProof(island1, []*integrationTests.TestProcessorNode{pA, metaNode}, round, nonce)
+		integrationTests.ProposeBlockWithProof(island1, []*integrationTests.TestProcessorNode{pA}, round, nonce)
 		time.Sleep(integrationTests.SyncDelay)
 
 		for _, target := range island2 {
 			mirrorCurrentBlock(target, pA)
+		}
+		time.Sleep(integrationTests.SyncDelay)
+
+		integrationTests.ProposeBlockWithProof(island2, []*integrationTests.TestProcessorNode{metaNode}, round, nonce)
+		time.Sleep(integrationTests.SyncDelay)
+
+		for _, target := range island1 {
 			mirrorCurrentBlock(target, metaNode)
 		}
 		time.Sleep(integrationTests.SyncDelay)
@@ -117,6 +149,9 @@ func TestSupernovaSync_ReconcileBackstop_FinalizedMinorityConverges(t *testing.T
 		nonce++
 	}
 
+	// meta built its own chain over the prefix and stops here; the shard nonce moves on without it
+	nextMetaNonce := nonce
+
 	// round 5: island 1 commits and instantly finalizes the clean sibling A
 	integrationTests.ProposeBlockWithProof(island1, []*integrationTests.TestProcessorNode{pA}, round, nonce)
 	time.Sleep(integrationTests.SyncDelay)
@@ -159,8 +194,33 @@ func TestSupernovaSync_ReconcileBackstop_FinalizedMinorityConverges(t *testing.T
 	round = integrationTests.IncrementAndPrintRound(round)
 	integrationTests.UpdateRound(allNodes, round)
 
-	// deliver the winning branch to island 1: its final head A is now a proven loser
+	// B is contended, so meta may only arbitrate it once the discovery window has elapsed
+	for i := 0; i < metaArbitrationWindowRounds; i++ {
+		round = integrationTests.IncrementAndPrintRound(round)
+		integrationTests.UpdateRound(allNodes, round)
+		time.Sleep(integrationTests.SyncDelay)
+	}
+
+	// meta arbitrates B, then extends itself so the notarizing block is settled and thus held final
+	integrationTests.ProposeBlockWithProof(island2, []*integrationTests.TestProcessorNode{metaNode}, round, nextMetaNonce)
+	time.Sleep(integrationTests.SyncDelay)
+	notarizingMeta, notarizingMetaHash, notarizingMetaProof := grabCurrentBlock(metaNode)
+	requireNotarizes(t, notarizingMeta, shardId, hashB, hashC)
+
+	round = integrationTests.IncrementAndPrintRound(round)
+	integrationTests.UpdateRound(allNodes, round)
+	integrationTests.ProposeBlockWithProof(island2, []*integrationTests.TestProcessorNode{metaNode}, round, nextMetaNonce+1)
+	time.Sleep(integrationTests.SyncDelay)
+	settlingMeta, settlingMetaHash, settlingMetaProof := grabCurrentBlock(metaNode)
+	require.Equal(t, string(notarizingMetaHash), string(settlingMeta.GetPrevHash()))
+
+	round = integrationTests.IncrementAndPrintRound(round)
+	integrationTests.UpdateRound(allNodes, round)
+
+	// deliver the winning branch AND the authority verdict on it to island 1
 	for _, n := range []*integrationTests.TestProcessorNode{pA, obsA} {
+		injectBlock(n, notarizingMeta, notarizingMetaHash, notarizingMetaProof)
+		injectBlock(n, settlingMeta, settlingMetaHash, settlingMetaProof)
 		injectBlock(n, headerB, hashB, proofB)
 		injectBlock(n, headerC, hashC, proofC)
 	}
@@ -183,6 +243,8 @@ func TestSupernovaSync_ReconcileBackstop_FinalizedMinorityConverges(t *testing.T
 		// the forced rollback clears pools above the rollback nonce; in production the
 		// majority network answers the re-requests -- modeled by re-injecting each round
 		for _, n := range []*integrationTests.TestProcessorNode{pA, obsA} {
+			injectBlock(n, notarizingMeta, notarizingMetaHash, notarizingMetaProof)
+			injectBlock(n, settlingMeta, settlingMetaHash, settlingMetaProof)
 			injectBlock(n, headerB, hashB, proofB)
 			injectBlock(n, headerC, hashC, proofC)
 		}
@@ -199,8 +261,9 @@ func TestSupernovaSync_ReconcileBackstop_FinalizedMinorityConverges(t *testing.T
 		assert.Equal(t, uint64(4), n.ForkDetector.GetHighestFinalBlockNonce())
 	}
 
-	// island 2 was never on the losing block and is untouched
-	for _, n := range island2 {
+	// island 2 was never on the losing block and is untouched (the meta node runs its own chain,
+	// so only the shard nodes are compared here)
+	for _, n := range []*integrationTests.TestProcessorNode{pB, obsB} {
 		assert.Equal(t, string(hashC), string(n.BlockChain.GetCurrentBlockHeaderHash()))
 		assert.Equal(t, uint64(4), n.ForkDetector.GetHighestFinalBlockNonce())
 	}

integrationTests/testSyncNode.go

@@ -29,6 +29,7 @@ import (
 	"github.com/multiversx/mx-chain-go/process/block"
 	"github.com/multiversx/mx-chain-go/process/block/bootstrapStorage"
 	"github.com/multiversx/mx-chain-go/process/sync"
+	"github.com/multiversx/mx-chain-go/process/track"
 	"github.com/multiversx/mx-chain-go/state"
 	stateDisabled "github.com/multiversx/mx-chain-go/state/disabled"
 	"github.com/multiversx/mx-chain-go/testscommon"
@@ -366,8 +367,17 @@ func (tpn *TestProcessorNode) createShardBootstrapper() (TestBootstrapper, error
 		ProcessConfigsHandler:        tpn.ProcessConfigsHandler,
 	}
 
+	metaFinalityView, err := track.NewMetaFinalityView(track.ArgsMetaFinalityView{
+		HeadersPool: tpn.DataPool.Headers(),
+		ProofsPool:  tpn.DataPool.Proofs(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	argsShardBootstrapper := sync.ArgShardBootstrapper{
 		ArgBaseBootstrapper: argsBaseBootstrapper,
+		MetaFinalityView:    metaFinalityView,
 	}
 
 	bootstrap, err := sync.NewShardBootstrap(argsShardBootstrapper)

process/errors.go

@@ -254,6 +254,9 @@ var ErrNilMetaBlocksPool = errors.New("nil meta blocks pool")
 // ErrNilProofsPool signals that a nil proofs pool was used
 var ErrNilProofsPool = errors.New("nil proofs pool")
 
+// ErrNilMetaFinalityView signals that a nil meta finality view was used
+var ErrNilMetaFinalityView = errors.New("nil meta finality view")
+
 // ErrNilTxProcessor signals that a nil transactions processor was used
 var ErrNilTxProcessor = errors.New("nil transactions processor")
 

process/sync/argBootstrapper.go

@@ -60,6 +60,7 @@ type ArgBaseBootstrapper struct {
 // new instances of shard bootstrapper
 type ArgShardBootstrapper struct {
 	ArgBaseBootstrapper
+	MetaFinalityView process.MetaFinalityView
 }
 
 // ArgMetaBootstrapper holds all dependencies required by the bootstrap data factory in order to create

process/sync/baseSync.go

@@ -81,9 +81,10 @@ type nonceRecoveryInfo struct {
 }
 
 type reconcileEvidence struct {
-	nonce          uint64
-	localHash      []byte
-	competitorHash []byte
+	nonce              uint64
+	localHash          []byte
+	competitorHash     []byte
+	lastEvaluatedRound int64
 }
 
 type baseBootstrap struct {
@@ -106,6 +107,7 @@ type baseBootstrap struct {
 	shardCoordinator    sharding.Coordinator
 	accounts            state.AccountsAdapter
 	blockBootstrapper   blockBootstrapper
+	settlementChecker   settlementChecker
 	blackListHandler    process.TimeCacher
 	enableEpochsHandler common.EnableEpochsHandler
 	enableRoundsHandler common.EnableRoundsHandler
@@ -1049,6 +1051,14 @@ func (boot *baseBootstrap) prepareForSyncAtBoostrapIfNeeded() error {
 
 func (boot *baseBootstrap) syncBlock() error {
 	boot.computeNodeState()
+
+	// evaluated before the synchronized gate: the authority may settle a childless competitor, and
+	// that leaves the node reading as synchronized while its final head is already dead
+	if boot.tryReconcileEquivocation() {
+		boot.invalidateNodeState()
+		return nil
+	}
+
 	nodeState := boot.GetNodeState()
 
 	if nodeState != common.NsNotSynchronized {
@@ -1061,15 +1071,7 @@ func (boot *baseBootstrap) syncBlock() error {
 		return nil
 	}
 
-	defer func() {
-		boot.mutNodeState.Lock()
-		boot.isNodeStateCalculated = false
-		boot.mutNodeState.Unlock()
-	}()
-
-	if boot.tryReconcileEquivocation() {
-		return nil
-	}
+	defer boot.invalidateNodeState()
 
 	if boot.forkInfo.IsDetected {
 		boot.statusHandler.Increment(common.MetricNumTimesInForkChoice)
@@ -2346,7 +2348,12 @@ func (boot *baseBootstrap) onEquivocationEvidence(headerProof data.HeaderProofHa
 	}
 
 	boot.mutReconcile.Lock()
-	boot.pendingReconcile = &reconcileEvidence{nonce: nonce, localHash: localHash, competitorHash: competitorHash}
+	boot.pendingReconcile = &reconcileEvidence{
+		nonce:              nonce,
+		localHash:          localHash,
+		competitorHash:     competitorHash,
+		lastEvaluatedRound: -1,
+	}
 	boot.mutReconcile.Unlock()
 
 	log.Warn("equivocation proof observed at the final head nonce, reconcile evidence recorded",
@@ -2372,38 +2379,33 @@ func pickCompetitorHash(localHash []byte, headerProof data.HeaderProofHandler, c
 	return nil
 }
 
-// tryReconcileEquivocation overrides the final gate and forces the switch when the final head is
-// childless and the equivocation competitor has a proofed child; never past a settled descendant
+// tryReconcileEquivocation overrides the final gate and forces the switch when the settlement
+// authority settled the equivocation competitor and not the local block
 func (boot *baseBootstrap) tryReconcileEquivocation() bool {
-	boot.mutReconcile.Lock()
-	evidence := boot.pendingReconcile
-	boot.mutReconcile.Unlock()
-
+	evidence, shouldEvaluate := boot.reconcileEvidenceToEvaluate()
 	if evidence == nil {
 		return false
 	}
 
-	currentHeader := boot.chainHandler.GetCurrentBlockHeader()
-	currentHash := boot.chainHandler.GetCurrentBlockHeaderHash()
-	stillApplies := !check.IfNil(currentHeader) &&
-		currentHeader.GetNonce() == evidence.nonce &&
-		bytes.Equal(currentHash, evidence.localHash) &&
-		evidence.nonce == boot.forkDetector.GetHighestFinalBlockNonce()
-	if !stillApplies {
+	if !boot.reconcileEvidenceStillApplies(evidence) {
 		boot.clearReconcileEvidence(evidence)
 		return false
 	}
 
-	if boot.hasProofedChild(evidence.nonce+1, evidence.localHash) {
+	if !shouldEvaluate {
+		return false
+	}
+
+	if boot.settlementChecker.isSettled(evidence.nonce, evidence.localHash) {
 		boot.clearReconcileEvidence(evidence)
 		return false
 	}
 
 	selfID := boot.shardCoordinator.SelfId()
 	isCompetitorSettled := boot.proofs.HasProof(selfID, evidence.competitorHash) &&
-		boot.hasProofedChild(evidence.nonce+1, evidence.competitorHash)
+		boot.settlementChecker.isSettled(evidence.nonce, evidence.competitorHash)
 	if !isCompetitorSettled {
-		// the settling child may still arrive; keep the evidence armed for the next iteration
+		// the authority's verdict may still arrive; keep the evidence armed for the next round
 		return false
 	}
 
@@ -2422,31 +2424,49 @@ func (boot *baseBootstrap) tryReconcileEquivocation() bool {
 	return true
 }
 
-func (boot *baseBootstrap) clearReconcileEvidence(evidence *reconcileEvidence) {
+// the settlement checks walk the pools, so the authority is consulted at most once per round
+func (boot *baseBootstrap) reconcileEvidenceToEvaluate() (*reconcileEvidence, bool) {
 	boot.mutReconcile.Lock()
-	if boot.pendingReconcile == evidence {
-		boot.pendingReconcile = nil
-	}
-	boot.mutReconcile.Unlock()
-}
+	defer boot.mutReconcile.Unlock()
 
-func (boot *baseBootstrap) hasProofedChild(nonce uint64, parentHash []byte) bool {
-	selfID := boot.shardCoordinator.SelfId()
-	headers, hashes, err := boot.headers.GetHeadersByNonceAndShardId(nonce, selfID)
-	if err != nil {
-		return false
+	evidence := boot.pendingReconcile
+	if evidence == nil {
+		return nil, false
 	}
 
-	for i, header := range headers {
-		if check.IfNil(header) || !bytes.Equal(header.GetPrevHash(), parentHash) {
-			continue
-		}
-		if boot.proofs.HasProof(selfID, hashes[i]) {
-			return true
-		}
+	currentRound := boot.roundHandler.Index()
+	shouldEvaluate := evidence.lastEvaluatedRound != currentRound
+	if shouldEvaluate {
+		evidence.lastEvaluatedRound = currentRound
 	}
 
-	return false
+	return evidence, shouldEvaluate
+}
+
+// invalidateNodeState forces the next iteration to recompute the fork info, so a roll back armed
+// here is picked up without waiting for the round to change
+func (boot *baseBootstrap) invalidateNodeState() {
+	boot.mutNodeState.Lock()
+	boot.isNodeStateCalculated = false
+	boot.mutNodeState.Unlock()
+}
+
+func (boot *baseBootstrap) reconcileEvidenceStillApplies(evidence *reconcileEvidence) bool {
+	currentHeader := boot.chainHandler.GetCurrentBlockHeader()
+	currentHash := boot.chainHandler.GetCurrentBlockHeaderHash()
+
+	return !check.IfNil(currentHeader) &&
+		currentHeader.GetNonce() == evidence.nonce &&
+		bytes.Equal(currentHash, evidence.localHash) &&
+		evidence.nonce == boot.forkDetector.GetHighestFinalBlockNonce()
+}
+
+func (boot *baseBootstrap) clearReconcileEvidence(evidence *reconcileEvidence) {
+	boot.mutReconcile.Lock()
+	if boot.pendingReconcile == evidence {
+		boot.pendingReconcile = nil
+	}
+	boot.mutReconcile.Unlock()
 }
 
 func (boot *baseBootstrap) isForcedRollBackOneBlock() bool {