From b31a9d7a681d8760f7d52058ce72e154289ba2e3 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 29 Jun 2026 13:15:53 +0500 Subject: [PATCH 01/11] fix --- .../query/stat/StatisticsConfigurationTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index eced0606697ef..f27664bf3fd67 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -153,6 +153,15 @@ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { protected void stopGridAndChangeBaseline(int nodeIdx) { stopGrid(nodeIdx); + // Wait for topology to stabilize before setting new baseline to avoid race condition + // where baseline is set while discovery messages about the stopped node are still propagating. + try { + awaitPartitionMapExchange(); + } + catch (InterruptedException e) { + // No-op. + } + if (persist) F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); From 71725752d4f7bb44d1568ed2351d441c607d090f Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 29 Jun 2026 13:44:36 +0500 Subject: [PATCH 02/11] minor --- .../stat/StatisticsConfigurationTest.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index f27664bf3fd67..4142af2110008 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -153,22 +153,23 @@ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { protected void stopGridAndChangeBaseline(int nodeIdx) { stopGrid(nodeIdx); - // Wait for topology to stabilize before setting new baseline to avoid race condition - // where baseline is set while discovery messages about the stopped node are still propagating. - try { - awaitPartitionMapExchange(); - } - catch (InterruptedException e) { - // No-op. - } + if (persist) { + // Wait for topology to stabilize before setting new baseline to avoid race condition + // where baseline is set while discovery messages about the stopped node are still propagating. + try { + awaitPartitionMapExchange(); + } + catch (InterruptedException ignored) { + // No-op. + } - if (persist) F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); + } try { awaitPartitionMapExchange(); } - catch (InterruptedException e) { + catch (InterruptedException ignored) { // No-op. } } From ee06c17e6e05d1b2e5698270b22d9788f4fb9374 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 29 Jun 2026 14:36:21 +0500 Subject: [PATCH 03/11] fix v2 --- .../processors/query/stat/StatisticsConfigurationTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index 4142af2110008..e2c1eec493c5e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -135,6 +135,11 @@ public static List parameters() { cleanPersistenceDir(); } + /** {@inheritDoc} */ + @Override protected long getPartitionMapExchangeTimeout() { + return super.getPartitionMapExchangeTimeout() * 2; + } + /** */ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { IgniteEx ign = startGrid(nodeIdx); From 4ff12edf9a3413fc44c3e4c905a29444e8ca8f83 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Mon, 29 Jun 2026 16:11:04 +0500 Subject: [PATCH 04/11] fix v3 --- .../processors/query/stat/StatisticsConfigurationTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index e2c1eec493c5e..708b308487439 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -169,6 +169,11 @@ protected void stopGridAndChangeBaseline(int nodeIdx) { } F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); + + // setBaselineTopology detects lost partitions but does NOT reset them. + // Reset them so owners are restored to OWNING state (awaitPartitionMapExchange relies on this). + for (Ignite grid : G.allGrids()) + grid.resetLostPartitions(((IgniteEx)grid).context().cache().cacheNames()); } try { From adb1b0e80685689909927de968e25c974b5c850e Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 30 Jun 2026 11:21:43 +0500 Subject: [PATCH 05/11] fix v4 --- .../stat/StatisticsConfigurationTest.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index 708b308487439..e36458fb29b48 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -155,25 +155,17 @@ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { } /** */ - protected void stopGridAndChangeBaseline(int nodeIdx) { + protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { stopGrid(nodeIdx); if (persist) { - // Wait for topology to stabilize before setting new baseline to avoid race condition - // where baseline is set while discovery messages about the stopped node are still propagating. - try { - awaitPartitionMapExchange(); - } - catch (InterruptedException ignored) { - // No-op. - } + // Wait for exchange to complete and all local partitions to reach OWNING state + // BEFORE changing baseline. If baseline is changed while partitions are still MOVING, + // the new exchange will detect them as lost (no owners yet) and trigger recovery scan. + awaitPartitionMapExchange(true, false, null); + // Now safe to update baseline without triggering lost partition detection. F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); - - // setBaselineTopology detects lost partitions but does NOT reset them. - // Reset them so owners are restored to OWNING state (awaitPartitionMapExchange relies on this). - for (Ignite grid : G.allGrids()) - grid.resetLostPartitions(((IgniteEx)grid).context().cache().cacheNames()); } try { From c346da7f3f3dd58084492a3fdee5c335a71bab84 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 2 Jul 2026 11:36:12 +0500 Subject: [PATCH 06/11] fix v5 --- .../query/stat/StatisticsConfigurationTest.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index e36458fb29b48..d36bd69dd44ac 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -137,7 +137,7 @@ public static List parameters() { /** {@inheritDoc} */ @Override protected long getPartitionMapExchangeTimeout() { - return super.getPartitionMapExchangeTimeout() * 2; + return super.getPartitionMapExchangeTimeout() * 3; } /** */ @@ -158,20 +158,13 @@ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { stopGrid(nodeIdx); - if (persist) { - // Wait for exchange to complete and all local partitions to reach OWNING state - // BEFORE changing baseline. If baseline is changed while partitions are still MOVING, - // the new exchange will detect them as lost (no owners yet) and trigger recovery scan. - awaitPartitionMapExchange(true, false, null); - - // Now safe to update baseline without triggering lost partition detection. + if (persist) F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); - } try { awaitPartitionMapExchange(); } - catch (InterruptedException ignored) { + catch (InterruptedException e) { // No-op. } } From 4345b2aa56184693f27e62d89d25c42d019b53ba Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 2 Jul 2026 11:37:18 +0500 Subject: [PATCH 07/11] fix v6 --- .../processors/query/stat/StatisticsConfigurationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index d36bd69dd44ac..a0cf9af8468f7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -155,7 +155,7 @@ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { } /** */ - protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { + protected void stopGridAndChangeBaseline(int nodeIdx) { stopGrid(nodeIdx); if (persist) From 3efa63912201d34984505099b5602baa2944f394 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Wed, 8 Jul 2026 12:41:05 +0500 Subject: [PATCH 08/11] fix v7 --- .../stat/StatisticsConfigurationTest.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index a0cf9af8468f7..7dc335429cbf6 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -155,17 +155,26 @@ protected IgniteEx startGridAndChangeBaseline(int nodeIdx) throws Exception { } /** */ - protected void stopGridAndChangeBaseline(int nodeIdx) { + protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { stopGrid(nodeIdx); - if (persist) + if (persist) { F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); - try { - awaitPartitionMapExchange(); + // Wait for all partitions to reach OWNING after baseline change. + // setBaselineTopology triggers a full exchange with async rebalancing. + // The regular awaitPartitionMapExchange() only waits for exchange completion, + // not for partitions to settle. If the next stopGrid() is called while + // partitions are still MOVING, a new exchange cascades on top → timeout. + awaitPartitionMapExchange(true, false, null); } - catch (InterruptedException e) { - // No-op. + else { + try { + awaitPartitionMapExchange(); + } + catch (InterruptedException e) { + // No-op. + } } } From fe55195f503b8ab8254911565d9f90a21b5c5f1d Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Wed, 8 Jul 2026 15:42:07 +0500 Subject: [PATCH 09/11] fix v8 --- .../stat/StatisticsConfigurationTest.java | 71 ++++++++++++++++--- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index 7dc335429cbf6..6bdb850ccb1ce 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -30,7 +30,9 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.query.stat.config.StatisticsObjectConfiguration; import org.apache.ignite.internal.processors.query.stat.messages.StatisticsObjectData; @@ -161,21 +163,68 @@ protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { if (persist) { F.first(G.allGrids()).cluster().setBaselineTopology(F.first(G.allGrids()).cluster().topologyVersion()); - // Wait for all partitions to reach OWNING after baseline change. + // Wait for rebalancing to finish on all nodes after baseline change. // setBaselineTopology triggers a full exchange with async rebalancing. - // The regular awaitPartitionMapExchange() only waits for exchange completion, - // not for partitions to settle. If the next stopGrid() is called while - // partitions are still MOVING, a new exchange cascades on top → timeout. - awaitPartitionMapExchange(true, false, null); - } - else { - try { - awaitPartitionMapExchange(); + // CacheAffinityChangeMessage may trigger a new exchange (higher minorTopVer) + // while rebalancing is still ongoing. awaitPartitionMapExchange checks + // ownerNodesCnt against the readyAffinityVersion, but partition topology + // (MOVING) lags behind — causing infinite loop and timeout. + for (Ignite grid : G.allGrids()) { + IgniteEx ign = (IgniteEx)grid; + + if (ign.cluster().localNode().isClient()) + continue; + + waitRebalanceFinishedNoVersionCheck(ign, "SMALLnull"); } - catch (InterruptedException e) { - // No-op. + } + + try { + awaitPartitionMapExchange(); + } + catch (InterruptedException e) { + // No-op. + } + } + + /** + * Wait for rebalance to finish without strict topology version check. + * Unlike {@link #waitRebalanceFinished}, this method simply waits for the rebalance future + * to complete, which is more robust when topology changes frequently (e.g., after + * setBaselineTopology). + */ + private void waitRebalanceFinishedNoVersionCheck(IgniteEx ignite, String cacheName) throws Exception { + long t0 = System.currentTimeMillis(); + long timeout = 30_000; + + // Poll a few times: rebalance may not have started yet (isInitial=true before exchange). + int initialPolls = 10; + + while (System.currentTimeMillis() - t0 < timeout) { + IgniteInternalFuture fut = ignite.cachex(cacheName).context().preloader().rebalanceFuture(); + + GridDhtPartitionDemander.RebalanceFuture rebFut = (GridDhtPartitionDemander.RebalanceFuture) fut; + + if (rebFut.isInitial()) { + if (initialPolls-- > 0) { + Thread.sleep(100); + continue; + } + // Rebalance still initial after polling — no rebalance needed. + return; } + + // Wait for rebalance to complete. + boolean res = fut.get(); + + if (res) + return; + + // Rebalance was cancelled (topology changed again) — retry with new future. + Thread.sleep(50); } + + throw new AssertionError("Rebalance did not finish within " + timeout + "ms for cache " + cacheName); } /** From e0b2ffe4173f95d0b0d50d293fe43adce84c3edc Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 9 Jul 2026 11:47:49 +0500 Subject: [PATCH 10/11] fix v9 --- .../stat/StatisticsConfigurationTest.java | 58 ++++--------------- 1 file changed, 12 insertions(+), 46 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index 6bdb850ccb1ce..9a987647829fb 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -30,7 +30,6 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; @@ -165,17 +164,24 @@ protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { // Wait for rebalancing to finish on all nodes after baseline change. // setBaselineTopology triggers a full exchange with async rebalancing. - // CacheAffinityChangeMessage may trigger a new exchange (higher minorTopVer) - // while rebalancing is still ongoing. awaitPartitionMapExchange checks - // ownerNodesCnt against the readyAffinityVersion, but partition topology - // (MOVING) lags behind — causing infinite loop and timeout. + // awaitPartitionMapExchange checks ownerNodesCnt against readyAffinityVersion, + // but partition topology (MOVING) lags behind — causing infinite loop and timeout. + // We wait for the rebalance future to complete (blocking.get()) before proceeding. for (Ignite grid : G.allGrids()) { IgniteEx ign = (IgniteEx)grid; if (ign.cluster().localNode().isClient()) continue; - waitRebalanceFinishedNoVersionCheck(ign, "SMALLnull"); + // Wait for ongoing rebalance to complete. + GridDhtPartitionDemander.RebalanceFuture rebFut = (GridDhtPartitionDemander.RebalanceFuture)ign + .cachex("SMALLnull").context().preloader().rebalanceFuture(); + + // If rebalance was cancelled (false), a new one may have started — retry. + while (!rebFut.isInitial() && !rebFut.get()) { + rebFut = (GridDhtPartitionDemander.RebalanceFuture) ign.cachex("SMALLnull") + .context().preloader().rebalanceFuture(); + } } } @@ -187,46 +193,6 @@ protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { } } - /** - * Wait for rebalance to finish without strict topology version check. - * Unlike {@link #waitRebalanceFinished}, this method simply waits for the rebalance future - * to complete, which is more robust when topology changes frequently (e.g., after - * setBaselineTopology). - */ - private void waitRebalanceFinishedNoVersionCheck(IgniteEx ignite, String cacheName) throws Exception { - long t0 = System.currentTimeMillis(); - long timeout = 30_000; - - // Poll a few times: rebalance may not have started yet (isInitial=true before exchange). - int initialPolls = 10; - - while (System.currentTimeMillis() - t0 < timeout) { - IgniteInternalFuture fut = ignite.cachex(cacheName).context().preloader().rebalanceFuture(); - - GridDhtPartitionDemander.RebalanceFuture rebFut = (GridDhtPartitionDemander.RebalanceFuture) fut; - - if (rebFut.isInitial()) { - if (initialPolls-- > 0) { - Thread.sleep(100); - continue; - } - // Rebalance still initial after polling — no rebalance needed. - return; - } - - // Wait for rebalance to complete. - boolean res = fut.get(); - - if (res) - return; - - // Rebalance was cancelled (topology changed again) — retry with new future. - Thread.sleep(50); - } - - throw new AssertionError("Rebalance did not finish within " + timeout + "ms for cache " + cacheName); - } - /** * Check statistics on cluster after change topology. * 1. Create statistic for a table; From 634274e8c084d88c1975d209a87b7b66ce074d1f Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Thu, 9 Jul 2026 12:31:12 +0500 Subject: [PATCH 11/11] fix v9 --- .../processors/query/stat/StatisticsConfigurationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java index 9a987647829fb..ba042bf2581e5 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/stat/StatisticsConfigurationTest.java @@ -179,7 +179,7 @@ protected void stopGridAndChangeBaseline(int nodeIdx) throws Exception { // If rebalance was cancelled (false), a new one may have started — retry. while (!rebFut.isInitial() && !rebFut.get()) { - rebFut = (GridDhtPartitionDemander.RebalanceFuture) ign.cachex("SMALLnull") + rebFut = (GridDhtPartitionDemander.RebalanceFuture)ign.cachex("SMALLnull") .context().preloader().rebalanceFuture(); } }