You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sk...@apache.org on 2024/12/16 17:55:42 UTC
(ignite-3) branch main updated: IGNITE-23780 Node restart behaviour when forced pending (#4847)
This is an automated email from the ASF dual-hosted git repository.
sk0x50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 6a4fafe855 IGNITE-23780 Node restart behaviour when forced pending (#4847)
6a4fafe855 is described below
commit 6a4fafe855859eaec5ca83280c6fdcb076cecb9d
Author: Cyrill <cy...@gmail.com>
AuthorDate: Mon Dec 16 20:55:35 2024 +0300
IGNITE-23780 Node restart behaviour when forced pending (#4847)
Co-authored-by: Kirill Sizov <si...@gmail.com>
---
.../distributionzones/DistributionZoneManager.java | 1 +
.../distributionzones/rebalance/RebalanceUtil.java | 24 +++
.../org/apache/ignite/internal/app/IgniteImpl.java | 5 +
.../ItDisasterRecoveryReconfigurationTest.java | 200 +++++++++++++++++--
.../internal/table/distributed/TableManager.java | 216 +++++++++++++--------
5 files changed, 344 insertions(+), 102 deletions(-)
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
index 048a49c6bf..86408d3820 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
@@ -181,6 +181,7 @@ public class DistributionZoneManager extends
/**
* The logical topology mapped to the MS revision.
+ * TODO: https://issues.apache.org/jira/browse/IGNITE-23561 get rid of this map (or properly clean up it).
*/
private final ConcurrentSkipListMap<Long, Set<NodeWithAttributes>> logicalTopologyByRevision = new ConcurrentSkipListMap<>();
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
index 8434986f4f..fe08e9585b 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
@@ -636,4 +636,28 @@ public class RebalanceUtil {
})
.collect(toList());
}
+
+ /**
+ * Returns table pending assignments for all table partitions from meta storage locally.
+ *
+ * @param metaStorageManager Meta storage manager.
+ * @param tableId Table id.
+ * @param numberOfPartitions Number of partitions.
+ * @param revision Revision.
+ * @return Future with table assignments as a value.
+ */
+ public static List<Assignments> tablePendingAssignmentsGetLocally(
+ MetaStorageManager metaStorageManager,
+ int tableId,
+ int numberOfPartitions,
+ long revision
+ ) {
+ return IntStream.range(0, numberOfPartitions)
+ .mapToObj(p -> {
+ Entry e = metaStorageManager.getLocally(pendingPartAssignmentsKey(new TablePartitionId(tableId, p)), revision);
+
+ return e != null ? Assignments.fromBytes(e.value()) : null;
+ })
+ .collect(toList());
+ }
}
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
index fe033dcfe0..735f8913a6 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
@@ -1589,6 +1589,11 @@ public class IgniteImpl implements Ignite {
return metricManager;
}
+ @TestOnly
+ public TableManager distributedTableManager() {
+ return distributedTblMgr;
+ }
+
/** {@inheritDoc} */
@Override
public IgniteTransactions transactions() {
diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
index 6b2a776a81..f223d9652d 100644
--- a/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
+++ b/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
@@ -18,12 +18,12 @@
package org.apache.ignite.internal.disaster;
import static java.lang.String.format;
+import static java.util.Collections.emptyList;
import static java.util.Collections.emptySet;
import static java.util.Map.of;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl;
import static org.apache.ignite.internal.TestWrappers.unwrapTableImpl;
import static org.apache.ignite.internal.TestWrappers.unwrapTableManager;
import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE;
@@ -99,6 +99,8 @@ import org.apache.ignite.internal.table.distributed.disaster.LocalPartitionState
import org.apache.ignite.lang.IgniteException;
import org.apache.ignite.raft.jraft.RaftGroupService;
import org.apache.ignite.raft.jraft.Status;
+import org.apache.ignite.raft.jraft.core.NodeImpl;
+import org.apache.ignite.raft.jraft.entity.LogId;
import org.apache.ignite.raft.jraft.error.RaftError;
import org.apache.ignite.raft.jraft.rpc.RpcRequests.AppendEntriesRequest;
import org.apache.ignite.raft.jraft.rpc.WriteActionRequest;
@@ -150,7 +152,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void setUp(TestInfo testInfo) throws Exception {
Method testMethod = testInfo.getTestMethod().orElseThrow();
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
zoneName = "ZONE_" + testMethod.getName().toUpperCase();
@@ -185,7 +187,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testInsertFailsIfMajorityIsLost() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(TABLE_NAME);
awaitPrimaryReplica(node0, partId);
@@ -232,7 +234,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testManualRebalanceIfMajorityIsLost() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(TABLE_NAME);
awaitPrimaryReplica(node0, partId);
@@ -270,7 +272,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
int fixingPartId = 1;
int anotherPartId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(TABLE_NAME);
awaitPrimaryReplica(node0, anotherPartId);
@@ -318,7 +320,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testManualRebalanceIfPartitionIsLost() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(TABLE_NAME);
awaitPrimaryReplica(node0, partId);
@@ -345,6 +347,149 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
assertThat(errors, is(empty()));
}
+ @Test
+ @ZoneParams(nodes = 5, replicas = 3, partitions = 1)
+ void testManualRebalanceRecovery() throws Exception {
+ int partId = 0;
+ // Disable scale down to avoid unwanted rebalance.
+ executeSql(format("ALTER ZONE %s SET data_nodes_auto_adjust_scale_down=%d", zoneName, INFINITE_TIMER_VALUE));
+
+ IgniteImpl node0 = igniteImpl(0);
+ int catalogVersion = node0.catalogManager().latestCatalogVersion();
+ long timestamp = node0.catalogManager().catalog(catalogVersion).time();
+ Table table = node0.tables().table(TABLE_NAME);
+
+ awaitPrimaryReplica(node0, partId);
+
+ Assignments allAssignments = Assignments.of(timestamp,
+ Assignment.forPeer(node(0).name()),
+ Assignment.forPeer(node(1).name()),
+ Assignment.forPeer(node(2).name())
+ );
+
+ // The partition is replicated to 3 nodes.
+ assertRealAssignments(node0, partId, 0, 1, 2);
+
+ List<Throwable> errors = insertValues(table, partId, 0);
+ assertThat(errors, is(empty()));
+
+ // Make sure node 0 has the same data as node 1 and 2.
+ assertTrue(
+ waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(1, partId)), SECONDS.toMillis(20)),
+ () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 1 log index= " + getRaftLogIndex(1, partId)
+ );
+ assertTrue(
+ waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(2, partId)), SECONDS.toMillis(20)),
+ () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 2 log index= " + getRaftLogIndex(2, partId)
+ );
+
+ // Stop 1 and 2, only 0 survived.
+ stopNodesInParallel(1, 2);
+
+ Assignments assignment0 = Assignments.of(timestamp,
+ Assignment.forPeer(node(0).name())
+ );
+
+ // Blocking stable switch to the first phase or reset,
+ // so that we'll have force pending assignments unexecuted.
+ blockMessage((nodeName, msg) -> stableKeySwitchMessage(msg, partId, assignment0));
+
+ // Init reset:
+ // pending = [0, force]
+ // planned = [0, 3, 4]
+ CompletableFuture<?> updateFuture = node0.disasterRecoveryManager().resetAllPartitions(
+ zoneName,
+ QUALIFIED_TABLE_NAME,
+ true
+ );
+
+ assertThat(updateFuture, willCompleteSuccessfully());
+
+ awaitPrimaryReplica(node0, partId);
+
+ // Since stable switch to 0 is blocked, data is stored on 0 node only.
+ assertRealAssignments(node0, partId, 0);
+
+ // And stable is in its initial state.
+ assertStableAssignments(node0, partId, allAssignments);
+
+ // Insert new data on the node 0.
+ errors = insertValues(table, partId, 10);
+ assertThat(errors, is(empty()));
+
+ // Start nodes 1 and 2 back, but they should not start their replicas.
+ startNode(1);
+ startNode(2);
+
+ // Make sure recovery has finished.
+ assertThat(igniteImpl(1).distributedTableManager().recoveryFuture(), willCompleteSuccessfully());
+ assertThat(igniteImpl(2).distributedTableManager().recoveryFuture(), willCompleteSuccessfully());
+
+ // Make sure 1 and 2 did not start.
+ assertRealAssignments(node0, partId, 0);
+ }
+
+ @Test
+ @ZoneParams(nodes = 4, replicas = 3, partitions = 1)
+ void testManualRebalanceRecoveryNoPending() throws Exception {
+ int partId = 0;
+
+ IgniteImpl node0 = igniteImpl(0);
+ int catalogVersion = node0.catalogManager().latestCatalogVersion();
+ long timestamp = node0.catalogManager().catalog(catalogVersion).time();
+ Table table = node0.tables().table(TABLE_NAME);
+
+ awaitPrimaryReplica(node0, partId);
+
+ Assignments allAssignments = Assignments.of(timestamp,
+ Assignment.forPeer(node(0).name()),
+ Assignment.forPeer(node(1).name()),
+ Assignment.forPeer(node(3).name())
+ );
+
+ // The partition is replicated to 3 nodes.
+ assertRealAssignments(node0, partId, 0, 1, 3);
+
+ List<Throwable> errors = insertValues(table, partId, 0);
+ assertThat(errors, is(empty()));
+
+ // Make sure node 0 has the same data as node 1 and 2.
+ assertTrue(
+ waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(1, partId)), SECONDS.toMillis(20)),
+ () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 1 log index= " + getRaftLogIndex(1, partId)
+ );
+ assertTrue(
+ waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(3, partId)), SECONDS.toMillis(20)),
+ () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 2 log index= " + getRaftLogIndex(3, partId)
+ );
+
+ Assignments assignmentPending = Assignments.of(timestamp,
+ Assignment.forPeer(node(0).name()),
+ Assignment.forPeer(node(1).name()),
+ Assignment.forPeer(node(2).name())
+ );
+
+ // Blocking stable switch to the first phase or reset,
+ // so that we'll have force pending assignments unexecuted.
+ blockMessage((nodeName, msg) -> stableKeySwitchMessage(msg, partId, assignmentPending));
+
+ // Stop 3. Nodes 0 and 1 survived.
+ stopNode(3);
+
+ waitForScale(node0, 3);
+
+ assertRealAssignments(node0, partId, 0, 1, 2);
+
+ assertPendingAssignments(node0, partId, assignmentPending);
+ assertStableAssignments(node0, partId, allAssignments);
+
+ // On start we check `pending.contains(node) || (stable.contains(node) && !pending.isForce())` condition.
+ // The node is in stable, not in pending and pending is not forced => start it.
+ startNode(3);
+
+ assertRealAssignments(node0, partId, 0, 1, 2, 3);
+ }
+
/**
* Tests a scenario where all stable nodes are lost, yet we have data on one of pending nodes and perform reset partition operation. In
* this case we should use that pending node as a source of data for recovery.
@@ -363,7 +508,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
public void testIncompleteRebalanceAfterResetPartitions() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
int catalogVersion = node0.catalogManager().latestCatalogVersion();
long timestamp = node0.catalogManager().catalog(catalogVersion).time();
@@ -379,7 +524,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
// Second snapshot causes log truncation.
triggerRaftSnapshot(1, partId);
- unwrapIgniteImpl(node(1)).dropMessages((nodeName, msg) -> {
+ igniteImpl(1).dropMessages((nodeName, msg) -> {
Ignite node = nullableNode(3);
return node != null && node.name().equals(nodeName) && msg instanceof SnapshotMvDataResponse;
@@ -454,7 +599,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testAutomaticRebalanceIfMajorityIsLost() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(TABLE_NAME);
awaitPrimaryReplica(node0, partId);
@@ -495,9 +640,9 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testAutomaticRebalanceIfPartitionIsLost() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
- IgniteImpl node1 = unwrapIgniteImpl(cluster.node(1));
+ IgniteImpl node1 = igniteImpl(1);
executeSql(format("ALTER ZONE %s SET data_nodes_auto_adjust_scale_down=%d", zoneName, INFINITE_TIMER_VALUE));
@@ -540,7 +685,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
public void testIncompleteRebalanceBeforeAutomaticResetPartitions() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
int catalogVersion = node0.catalogManager().latestCatalogVersion();
long timestamp = node0.catalogManager().catalog(catalogVersion).time();
@@ -610,7 +755,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testThoPhaseResetMaxLogIndex() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
int catalogVersion = node0.catalogManager().latestCatalogVersion();
long timestamp = node0.catalogManager().catalog(catalogVersion).time();
Table table = node0.tables().table(TABLE_NAME);
@@ -748,7 +893,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testThoPhaseResetEqualLogIndex() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
int catalogVersion = node0.catalogManager().latestCatalogVersion();
long timestamp = node0.catalogManager().catalog(catalogVersion).time();
Table table = node0.tables().table(TABLE_NAME);
@@ -870,7 +1015,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
void testTwoPhaseResetOnEmptyNodes() throws Exception {
int partId = 0;
- IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+ IgniteImpl node0 = igniteImpl(0);
int catalogVersion = node0.catalogManager().latestCatalogVersion();
long timestamp = node0.catalogManager().catalog(catalogVersion).time();
@@ -1007,7 +1152,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
}
private String findLeader(int nodeIdx, int partId) {
- IgniteImpl node = unwrapIgniteImpl(node(nodeIdx));
+ IgniteImpl node = igniteImpl(nodeIdx);
var raftNodeId = new RaftNodeId(new TablePartitionId(tableId, partId), new Peer(node.name()));
var jraftServer = (JraftServerImpl) node.raftManager().server();
@@ -1017,9 +1162,24 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
return raftGroupService.getRaftNode().getLeaderId().getConsistentId();
}
+ private NodeImpl getRaftNode(int nodeIdx, int partId) {
+ IgniteImpl node = igniteImpl(nodeIdx);
+
+ var raftNodeId = new RaftNodeId(new TablePartitionId(tableId, partId), new Peer(node.name()));
+ var jraftServer = (JraftServerImpl) node.raftManager().server();
+
+ RaftGroupService raftGroupService = jraftServer.raftGroupService(raftNodeId);
+ assertNotNull(raftGroupService);
+
+ return (NodeImpl) raftGroupService.getRaftNode();
+ }
+
+ private LogId getRaftLogIndex(int nodeIdx, int partId) {
+ return getRaftNode(nodeIdx, partId).lastLogIndexAndTerm();
+ }
+
private void triggerRaftSnapshot(int nodeIdx, int partId) throws InterruptedException, ExecutionException {
- //noinspection resource
- IgniteImpl node = unwrapIgniteImpl(node(nodeIdx));
+ IgniteImpl node = igniteImpl(nodeIdx);
var raftNodeId = new RaftNodeId(new TablePartitionId(tableId, partId), new Peer(node.name()));
var jraftServer = (JraftServerImpl) node.raftManager().server();
@@ -1158,6 +1318,10 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
LocalPartitionStateByNode partitionStates = partitionStatesFut.join().get(new TablePartitionId(tableId, partId));
+ if (partitionStates == null) {
+ return emptyList();
+ }
+
return partitionStates.keySet()
.stream()
.map(cluster::nodeIndex)
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index 21d5968f32..4c054ffba4 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -39,6 +39,7 @@ import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUt
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.stablePartAssignmentsKey;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.subtract;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.tableAssignmentsGetLocally;
+import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.tablePendingAssignmentsGetLocally;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.union;
import static org.apache.ignite.internal.event.EventListener.fromConsumer;
import static org.apache.ignite.internal.hlc.HybridTimestamp.LOGICAL_TIME_BITS_SIZE;
@@ -294,7 +295,8 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
/**
* Versioned value for tracking RAFT groups initialization and starting completion.
*
- * <p>Only explicitly updated in {@link #startLocalPartitionsAndClients(CompletableFuture, TableImpl, int, boolean, long)}.
+ * <p>Only explicitly updated in
+ * {@link #startLocalPartitionsAndClients(CompletableFuture, List, TableImpl, boolean, long)}.
*
* <p>Completed strictly after {@link #localPartitionsVv}.
*/
@@ -425,6 +427,8 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
private final Map<Integer, Set<TableImpl>> tablesPerZone = new ConcurrentHashMap<>();
+ private final CompletableFuture<Void> recoveryFuture = new CompletableFuture<>();
+
/**
* Creates a new table manager.
*
@@ -658,6 +662,11 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
});
}
+ @TestOnly
+ public CompletableFuture<Void> recoveryFuture() {
+ return recoveryFuture;
+ }
+
private CompletableFuture<Void> waitForMetadataCompleteness(long ts) {
return executorInclinedSchemaSyncService.waitForMetadataCompleteness(HybridTimestamp.hybridTimestamp(ts));
}
@@ -738,7 +747,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
CatalogTableDescriptor tableDescriptor,
boolean onNodeRecovery
) {
- TableImpl table = createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
+ TableImpl table = createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
int tableId = tableDescriptor.id();
@@ -896,7 +905,13 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
recoveryRevision,
(entry, rev) -> handleChangePendingAssignmentEvent(entry, rev, true),
"pending"
- ));
+ )).whenComplete((v, th) -> {
+ if (th != null) {
+ recoveryFuture.completeExceptionally(th);
+ } else {
+ recoveryFuture.complete(null);
+ }
+ });
}
private CompletableFuture<Void> handleAssignmentsOnRecovery(
@@ -1065,53 +1080,71 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
/**
* Updates or creates partition raft groups and storages.
*
- * @param assignmentsFuture Table assignments.
+ * @param stableAssignmentsFuture Table assignments.
* @param table Initialized table entity.
- * @param zoneId Zone id.
* @param isRecovery {@code true} if the node is being started up.
* @param assignmentsTimestamp Assignments timestamp.
* @return future, which will be completed when the partitions creations done.
*/
private CompletableFuture<Void> startLocalPartitionsAndClients(
- CompletableFuture<List<Assignments>> assignmentsFuture,
+ CompletableFuture<List<Assignments>> stableAssignmentsFuture,
+ List<@Nullable Assignments> pendingAssignmentsForPartitions,
TableImpl table,
- int zoneId,
boolean isRecovery,
long assignmentsTimestamp
) {
int tableId = table.tableId();
// Create new raft nodes according to new assignments.
- return assignmentsFuture.thenCompose(tableAssignments -> {
+ return stableAssignmentsFuture.thenCompose(stableAssignmentsForPartitions -> {
// Empty assignments might be a valid case if tables are created from within cluster init HOCON
// configuration, which is not supported now.
- assert tableAssignments != null : IgniteStringFormatter.format("Table [id={}] has empty assignments.", tableId);
+ assert stableAssignmentsForPartitions != null
+ : IgniteStringFormatter.format("Table [id={}] has empty assignments.", tableId);
- int partitions = tableAssignments.size();
+ int partitions = stableAssignmentsForPartitions.size();
var futures = new CompletableFuture<?>[partitions];
for (int i = 0; i < partitions; i++) {
int partId = i;
- Assignments assignments = tableAssignments.get(i);
+ Assignments stableAssignments = stableAssignmentsForPartitions.get(i);
- CompletableFuture<?> future = startPartitionAndStartClient(
- table,
- partId,
- localMemberAssignment(assignments),
- assignments,
- zoneId,
- isRecovery,
- assignmentsTimestamp
- )
- .whenComplete((res, ex) -> {
- if (ex != null) {
- LOG.warn("Unable to update raft groups on the node [tableId={}, partitionId={}]", ex, tableId, partId);
- }
- });
+ Assignments pendingAssignments = pendingAssignmentsForPartitions.get(i);
+
+ Assignment localMemberAssignmentInStable = localMemberAssignment(stableAssignments);
+
+ boolean shouldStartPartition;
- futures[i] = future;
+ if (isRecovery) {
+ // The condition to start the replica is
+ // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+ // However we check only the right part of this condition here
+ // since after `startTables` we have a call to `processAssignmentsOnRecovery`,
+ // which executes pending assignments update and will start required partitions there.
+ shouldStartPartition = localMemberAssignmentInStable != null
+ && (pendingAssignments == null || !pendingAssignments.force());
+ } else {
+ shouldStartPartition = localMemberAssignmentInStable != null;
+ }
+
+ if (shouldStartPartition) {
+ futures[i] = startPartitionAndStartClient(
+ table,
+ partId,
+ localMemberAssignmentInStable,
+ stableAssignments,
+ isRecovery,
+ assignmentsTimestamp
+ ).whenComplete((res, ex) -> {
+ if (ex != null) {
+ LOG.warn("Unable to update raft groups on the node [tableId={}, partitionId={}]", ex, tableId, partId);
+ }
+ });
+ } else {
+ futures[i] = nullCompletedFuture();
+ }
}
return allOf(futures);
@@ -1121,17 +1154,11 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
private CompletableFuture<Void> startPartitionAndStartClient(
TableImpl table,
int partId,
- @Nullable Assignment localMemberAssignment,
+ Assignment localMemberAssignment,
Assignments stableAssignments,
- int zoneId,
boolean isRecovery,
long assignmentsTimestamp
) {
- if (localMemberAssignment == null) {
- // (0) in case if node not in the assignments
- return nullCompletedFuture();
- }
-
int tableId = table.tableId();
var internalTbl = (InternalTableImpl) table.internalTable();
@@ -1244,10 +1271,10 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
}
@Nullable
- private Assignment localMemberAssignment(Assignments assignments) {
+ private Assignment localMemberAssignment(@Nullable Assignments assignments) {
Assignment localMemberAssignment = Assignment.forPeer(localNode().name());
- return assignments.nodes().contains(localMemberAssignment) ? localMemberAssignment : null;
+ return assignments != null && assignments.nodes().contains(localMemberAssignment) ? localMemberAssignment : null;
}
private PartitionMover createPartitionMover(TablePartitionId replicaGrpId) {
@@ -1570,15 +1597,19 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
// Retrieve descriptor during synchronous call, before the previous catalog version could be concurrently compacted.
CatalogZoneDescriptor zoneDescriptor = getZoneDescriptor(tableDescriptor, catalogVersion);
- CompletableFuture<List<Assignments>> assignmentsFuture = getOrCreateAssignments(
+ // Returns stable assignments.
+ CompletableFuture<List<Assignments>> stableAssignmentsFuture = getOrCreateAssignments(
tableDescriptor,
zoneDescriptor,
causalityToken,
catalogVersion
);
- CompletableFuture<List<Assignments>> assignmentsFutureAfterInvoke =
- writeTableAssignmentsToMetastore(tableId, assignmentsFuture);
+ List<Assignments> pendingAssignments =
+ tablePendingAssignmentsGetLocally(metaStorageMgr, tableId, zoneDescriptor.partitions(), causalityToken);
+
+ CompletableFuture<List<Assignments>> stableAssignmentsFutureAfterInvoke =
+ writeTableAssignmentsToMetastore(tableId, stableAssignmentsFuture);
Catalog catalog = catalogService.catalog(catalogVersion);
@@ -1586,7 +1617,8 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
causalityToken,
tableDescriptor,
zoneDescriptor,
- assignmentsFutureAfterInvoke,
+ stableAssignmentsFutureAfterInvoke,
+ pendingAssignments,
onNodeRecovery,
catalog.time()
);
@@ -1599,7 +1631,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
* @param causalityToken Causality token.
* @param tableDescriptor Catalog table descriptor.
* @param zoneDescriptor Catalog distributed zone descriptor.
- * @param assignmentsFuture Future with assignments.
+ * @param stableAssignmentsFuture Future with assignments.
* @param onNodeRecovery {@code true} when called during node recovery, {@code false} otherwise.
* @return Future that will be completed when local changes related to the table creation are applied.
*/
@@ -1607,11 +1639,12 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
long causalityToken,
CatalogTableDescriptor tableDescriptor,
CatalogZoneDescriptor zoneDescriptor,
- CompletableFuture<List<Assignments>> assignmentsFuture,
+ CompletableFuture<List<Assignments>> stableAssignmentsFuture,
+ List<Assignments> pendingAssignments,
boolean onNodeRecovery,
long assignmentsTimestamp
) {
- TableImpl table = createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
+ TableImpl table = createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
int tableId = tableDescriptor.id();
@@ -1625,7 +1658,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
// NB: all vv.update() calls must be made from the synchronous part of the method (not in thenCompose()/etc!).
CompletableFuture<?> localPartsUpdateFuture = localPartitionsVv.update(causalityToken,
- (ignore, throwable) -> inBusyLock(busyLock, () -> assignmentsFuture.thenComposeAsync(newAssignments -> {
+ (ignore, throwable) -> inBusyLock(busyLock, () -> stableAssignmentsFuture.thenComposeAsync(newAssignments -> {
PartitionSet parts = new BitSetPartitionSet();
for (int i = 0; i < newAssignments.size(); i++) {
@@ -1647,6 +1680,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
return failedFuture(e);
}
+ // `stableAssignmentsFuture` is already completed by this time (it's in chain of `localPartsUpdateFuture`).
return allOf(localPartsUpdateFuture, tablesByIdFuture).thenComposeAsync(ignore -> inBusyLock(busyLock, () -> {
if (onNodeRecovery) {
SchemaRegistry schemaRegistry = table.schemaView();
@@ -1657,9 +1691,9 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
registerIndexesToTable(table, catalogService, partitionSet, schemaRegistry, lwm);
}
return startLocalPartitionsAndClients(
- assignmentsFuture,
+ stableAssignmentsFuture,
+ pendingAssignments,
table,
- zoneDescriptor.id(),
onNodeRecovery,
assignmentsTimestamp
);
@@ -1693,17 +1727,21 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
assignmentsFuture = completedFuture(
tableAssignmentsGetLocally(metaStorageMgr, tableId, zoneDescriptor.partitions(), causalityToken));
} else {
-
Catalog catalog = catalogService.catalog(catalogVersion);
long assignmentsTimestamp = catalog.time();
assignmentsFuture = distributionZoneManager.dataNodes(causalityToken, catalogVersion, zoneDescriptor.id())
- .thenApply(dataNodes -> PartitionDistributionUtils.calculateAssignments(
- dataNodes,
- zoneDescriptor.partitions(),
- zoneDescriptor.replicas()
- ).stream().map(assignments -> Assignments.of(assignments, assignmentsTimestamp)).collect(toList()));
+ .thenApply(dataNodes ->
+ PartitionDistributionUtils.calculateAssignments(
+ dataNodes,
+ zoneDescriptor.partitions(),
+ zoneDescriptor.replicas()
+ )
+ .stream()
+ .map(assignments -> Assignments.of(assignments, assignmentsTimestamp))
+ .collect(toList())
+ );
assignmentsFuture.thenAccept(assignmentsList -> LOG.info(
"Assignments calculated from data nodes [table={}, tableId={}, assignments={}, revision={}]",
@@ -2097,16 +2135,13 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
);
}
- long assignmentsTimestamp = pendingAssignments.timestamp();
-
return handleChangePendingAssignmentEvent(
replicaGrpId,
table,
stableAssignments,
pendingAssignments,
revision,
- isRecovery,
- assignmentsTimestamp
+ isRecovery
).thenAccept(v -> executeIfLocalNodeIsPrimaryForGroup(
replicaGrpId,
replicaMeta -> sendChangePeersAndLearnersRequest(
@@ -2129,17 +2164,26 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
@Nullable Assignments stableAssignments,
Assignments pendingAssignments,
long revision,
- boolean isRecovery,
- long assignmentsTimestamp
+ boolean isRecovery
) {
boolean pendingAssignmentsAreForced = pendingAssignments.force();
Set<Assignment> pendingAssignmentsNodes = pendingAssignments.nodes();
+ long assignmentsTimestamp = pendingAssignments.timestamp();
// Start a new Raft node and Replica if this node has appeared in the new assignments.
- Assignment localMemberAssignment = localMemberAssignment(pendingAssignments);
-
- boolean shouldStartLocalGroupNode = localMemberAssignment != null
- && (stableAssignments == null || !stableAssignments.nodes().contains(localMemberAssignment));
+ Assignment localMemberAssignmentInPending = localMemberAssignment(pendingAssignments);
+ Assignment localMemberAssignmentInStable = localMemberAssignment(stableAssignments);
+
+ boolean shouldStartLocalGroupNode;
+ if (isRecovery) {
+ // The condition to start the replica is
+ // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+ // This condition covers the left part of the OR expression.
+ // The right part of it is covered in `startLocalPartitionsAndClients`.
+ shouldStartLocalGroupNode = localMemberAssignmentInPending != null;
+ } else {
+ shouldStartLocalGroupNode = localMemberAssignmentInPending != null && localMemberAssignmentInStable == null;
+ }
// This is a set of assignments for nodes that are not the part of stable assignments, i.e. unstable part of the distribution.
// For regular pending assignments we use (old) stable set, so that none of new nodes would be able to propose itself as a leader.
@@ -2152,7 +2196,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
if (stableAssignments == null || stableAssignments.nodes().isEmpty()) {
// This condition can only pass if all stable nodes are dead, and we start new raft group from scratch.
// In this case new initial configuration must match new forced assignments.
- computedStableAssignments = Assignments.forced(pendingAssignmentsNodes, pendingAssignments.timestamp());
+ computedStableAssignments = Assignments.forced(pendingAssignmentsNodes, assignmentsTimestamp);
} else if (pendingAssignmentsAreForced) {
// In case of forced assignments we need to remove nodes that are present in the stable set but are missing from the
// pending set. Such operation removes dead stable nodes from the resulting stable set, which guarantees that we will
@@ -2188,22 +2232,19 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
);
return waitForMetadataCompleteness(assignmentsTimestamp).thenCompose(ignored -> inBusyLock(busyLock, () -> {
- int catalogVersion = catalogService.activeCatalogVersion(assignmentsTimestamp);
-
- int zoneId = getTableDescriptor(tbl.tableId(), catalogVersion).zoneId();
+ assert localMemberAssignmentInPending != null : "Local member assignment";
return startPartitionAndStartClient(
tbl,
replicaGrpId.partitionId(),
- localMemberAssignment,
+ localMemberAssignmentInPending,
computedStableAssignments,
- zoneId,
isRecovery,
assignmentsTimestamp
);
}));
}), ioExecutor);
- } else if (pendingAssignmentsAreForced && localMemberAssignment != null) {
+ } else if (pendingAssignmentsAreForced && localMemberAssignmentInPending != null) {
localServicesStartFuture = runAsync(() -> inBusyLock(busyLock, () -> {
assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group: " + replicaGrpId;
@@ -2231,6 +2272,21 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
: "The local node is outside of the replication group [inStableOrPending=" + isLocalNodeInStableOrPending
+ ", isLeaseholder=" + isLeaseholder + "].";
+ // A node might exist in stable yet we don't want to start the replica
+ // The case is: nodes A, B and C hold a replication group,
+ // nodes A and B die.
+ // Reset adds C into force pending and user writes data onto C.
+ // Then A and B go back online. In this case
+ // stable = [A, B, C], pending = [C, force] and only C should be started.
+ if (isRecovery && !replicaMgr.isReplicaStarted(replicaGrpId)) {
+ return;
+ }
+
+ assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
+ + ", stable=" + stableAssignments
+ + ", pending=" + pendingAssignments
+ + ", localName=" + localNode().name() + "].";
+
// For forced assignments, we exclude dead stable nodes, and all alive stable nodes are already in pending assignments.
// Union is not required in such a case.
Set<Assignment> newAssignments = pendingAssignmentsAreForced || stableAssignments == null
@@ -2294,7 +2350,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
Assignments pendingAssignments,
long revision
) {
- Entry reduceEntry = metaStorageMgr.getLocally(RebalanceUtil.switchReduceKey(replicaGrpId), revision);
+ Entry reduceEntry = metaStorageMgr.getLocally(RebalanceUtil.switchReduceKey(replicaGrpId), revision);
Assignments reduceAssignments = reduceEntry != null
? Assignments.fromBytes(reduceEntry.value())
@@ -2304,17 +2360,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
? subtract(stableAssignments.nodes(), reduceAssignments.nodes())
: stableAssignments.nodes();
- if (!isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()))) {
- return false;
- }
-
- assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
- + ", stable=" + stableAssignments
- + ", pending=" + pendingAssignments
- + ", reduce=" + reduceAssignments
- + ", localName=" + localNode().name() + "].";
-
- return true;
+ return isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()));
}
private SnapshotStorageFactory createSnapshotStorageFactory(
@@ -2883,16 +2929,18 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
assert stableAssignments != null : "tablePartitionId=" + tablePartitionId + ", revision=" + revision;
return waitForMetadataCompleteness(assignmentsTimestamp).thenCompose(unused2 -> inBusyLockAsync(busyLock, () -> {
- int catalogVersion = catalogService.activeCatalogVersion(assignmentsTimestamp);
+ Assignment localMemberAssignment = localMemberAssignment(stableAssignments);
- int zoneId = getTableDescriptor(tablePartitionId.tableId(), catalogVersion).zoneId();
+ if (localMemberAssignment == null) {
+ // (0) in case if node not in the assignments
+ return nullCompletedFuture();
+ }
return startPartitionAndStartClient(
table,
tablePartitionId.partitionId(),
- localMemberAssignment(stableAssignments),
+ localMemberAssignment,
stableAssignments,
- zoneId,
false,
assignmentsTimestamp
);