You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sa...@apache.org on 2025/04/30 16:01:27 UTC
(ignite-3) branch main updated: IGNITE-24232 Implement correct node restart behavior when forced pending (#5726)
This is an automated email from the ASF dual-hosted git repository.
sanpwc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new ccf40c3fdc5 IGNITE-24232 Implement correct node restart behavior when forced pending (#5726)
ccf40c3fdc5 is described below
commit ccf40c3fdc5508fa09b62c9e950e1622f4b0f42f
Author: Cyrill <cy...@gmail.com>
AuthorDate: Wed Apr 30 19:01:21 2025 +0300
IGNITE-24232 Implement correct node restart behavior when forced pending (#5726)
---
.../rebalance/ZoneRebalanceUtil.java | 37 +++++-
.../PartitionReplicaLifecycleManager.java | 136 ++++++++++++++-------
.../disaster/ItDisasterRecoveryManagerTest.java | 1 +
.../ItDisasterRecoveryReconfigurationTest.java | 2 -
4 files changed, 123 insertions(+), 53 deletions(-)
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
index abe21910c15..c49519e7fb9 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
@@ -577,9 +577,9 @@ public class ZoneRebalanceUtil {
int partitionNumber,
long revision
) {
- Entry entry = metaStorageManager.getLocally(stablePartAssignmentsKey(new ZonePartitionId(zoneId, partitionNumber)), revision);
-
- return (entry == null || entry.empty() || entry.tombstone()) ? null : Assignments.fromBytes(entry.value()).nodes();
+ Assignments assignments =
+ zoneStableAssignmentsGetLocally(metaStorageManager, new ZonePartitionId(zoneId, partitionNumber), revision);
+ return assignments == null ? null : assignments.nodes();
}
/**
@@ -599,11 +599,36 @@ public class ZoneRebalanceUtil {
) {
return IntStream.range(0, numberOfPartitions)
.mapToObj(p -> {
- Entry e = metaStorageManager.getLocally(stablePartAssignmentsKey(new ZonePartitionId(zoneId, p)), revision);
+ Assignments assignments =
+ zoneStableAssignmentsGetLocally(metaStorageManager, new ZonePartitionId(zoneId, p), revision);
+
+ assert assignments != null : "No assignments found for " + new ZonePartitionId(zoneId, p);
+
+ return assignments;
+ })
+ .collect(toList());
+ }
- assert e != null && !e.empty() && !e.tombstone() : e;
+ /**
+ * Returns zone pending assignments for all zone partitions from meta storage locally. Assignments must be present.
+ *
+ * @param metaStorageManager Meta storage manager.
+ * @param zoneId Zone id.
+ * @param numberOfPartitions Number of partitions.
+ * @param revision Revision.
+ * @return Future with zone assignments as a value.
+ */
+ public static List<@Nullable Assignments> zonePendingAssignmentsGetLocally(
+ MetaStorageManager metaStorageManager,
+ int zoneId,
+ int numberOfPartitions,
+ long revision
+ ) {
+ return IntStream.range(0, numberOfPartitions)
+ .mapToObj(p -> {
+ Entry e = metaStorageManager.getLocally(pendingPartAssignmentsQueueKey(new ZonePartitionId(zoneId, p)), revision);
- return Assignments.fromBytes(e.value());
+ return e != null && !e.empty() && !e.tombstone() ? AssignmentsQueue.fromBytes(e.value()).poll() : null;
})
.collect(toList());
}
diff --git a/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java b/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java
index afa2cc361a1..870fcd6c8fd 100644
--- a/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java
+++ b/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java
@@ -38,6 +38,7 @@ import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalan
import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.stablePartAssignmentsKey;
import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zoneAssignmentsGetLocally;
import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zonePartitionAssignmentsGetLocally;
+import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zonePendingAssignmentsGetLocally;
import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zoneStableAssignmentsGetLocally;
import static org.apache.ignite.internal.hlc.HybridTimestamp.LOGICAL_TIME_BITS_SIZE;
import static org.apache.ignite.internal.hlc.HybridTimestamp.hybridTimestamp;
@@ -462,7 +463,7 @@ public class PartitionReplicaLifecycleManager extends
return handleAssignmentsOnRecovery(
new ByteArray(PENDING_ASSIGNMENTS_QUEUE_PREFIX_BYTES),
recoveryRevision,
- this::handleChangePendingAssignmentEvent,
+ (pendingAssignmentsEntry, revision) -> handleChangePendingAssignmentEvent(pendingAssignmentsEntry, revision, true),
"pending"
);
}
@@ -525,9 +526,10 @@ public class PartitionReplicaLifecycleManager extends
return getOrCreateAssignments(zoneDescriptor, causalityToken, catalogVersion)
.thenCompose(assignments -> writeZoneAssignmentsToMetastore(zoneId, assignments))
.thenCompose(
- assignments -> createZoneReplicationNodes(
+ stableAssignments -> createZoneReplicationNodes(
zoneId,
- assignments,
+ stableAssignments,
+ zonePendingAssignmentsGetLocally(metaStorageMgr, zoneId, zoneDescriptor.partitions(), causalityToken),
causalityToken,
catalogVersion,
zoneDescriptor.partitions(),
@@ -539,33 +541,55 @@ public class PartitionReplicaLifecycleManager extends
private CompletableFuture<Void> createZoneReplicationNodes(
int zoneId,
- List<Assignments> assignments,
+ List<Assignments> stableAssignmentsForZone,
+ List<@Nullable Assignments> pendingAssignmentsForZone,
long causalityToken,
int catalogVersion,
int partitionCount,
boolean onNodeRecovery
) {
- assert assignments != null : IgniteStringFormatter.format("Zone has empty assignments [id={}].", zoneId);
+ assert stableAssignmentsForZone != null : IgniteStringFormatter.format("Zone has empty assignments [id={}].", zoneId);
Supplier<CompletableFuture<Void>> createZoneReplicationNodes = () -> inBusyLockAsync(busyLock, () -> {
- var partitionsStartFutures = new CompletableFuture<?>[assignments.size()];
+ var partitionsStartFutures = new CompletableFuture<?>[stableAssignmentsForZone.size()];
- for (int partId = 0; partId < assignments.size(); partId++) {
- Assignments zoneAssignment = assignments.get(partId);
+ for (int partId = 0; partId < stableAssignmentsForZone.size(); partId++) {
+ Assignments stableAssignments = stableAssignmentsForZone.get(partId);
- Assignment localAssignment = localAssignment(zoneAssignment);
+ Assignments pendingAssignments = pendingAssignmentsForZone.get(partId);
- var zonePartitionId = new ZonePartitionId(zoneId, partId);
+ Assignment localAssignment = localAssignment(stableAssignments);
+
+ boolean shouldStartPartition;
+
+ if (onNodeRecovery) {
+ // The condition to start the replica is
+ // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+ // However we check only the right part of this condition here
+ // since after `startTables` we have a call to `processAssignmentsOnRecovery`,
+ // which executes pending assignments update and will start required partitions there.
+ shouldStartPartition = localAssignment != null
+ && (pendingAssignments == null || !pendingAssignments.force());
+ } else {
+ shouldStartPartition = localAssignment != null;
+ }
+
+ if (shouldStartPartition) {
+ var zonePartitionId = new ZonePartitionId(zoneId, partId);
+
+ partitionsStartFutures[partId] = createZonePartitionReplicationNode(
+ zonePartitionId,
+ localAssignment,
+ stableAssignments,
+ causalityToken,
+ partitionCount,
+ isVolatileZoneForCatalogVersion(zoneId, catalogVersion),
+ !onNodeRecovery
+ );
+ } else {
+ partitionsStartFutures[partId] = nullCompletedFuture();
+ }
- partitionsStartFutures[partId] = createZonePartitionReplicationNode(
- zonePartitionId,
- localAssignment,
- zoneAssignment,
- causalityToken,
- partitionCount,
- isVolatileZoneForCatalogVersion(zoneId, catalogVersion),
- !onNodeRecovery
- );
}
return allOf(partitionsStartFutures);
@@ -620,17 +644,13 @@ public class PartitionReplicaLifecycleManager extends
*/
private CompletableFuture<?> createZonePartitionReplicationNode(
ZonePartitionId zonePartitionId,
- @Nullable Assignment localAssignment,
+ Assignment localAssignment,
Assignments stableAssignments,
long revision,
int partitionCount,
boolean isVolatileZone,
boolean holdingZoneWriteLock
) {
- if (localAssignment == null) {
- return nullCompletedFuture();
- }
-
// TODO: https://issues.apache.org/jira/browse/IGNITE-22522 We need to integrate PartitionReplicatorNodeRecovery logic here when
// we in the recovery phase.
Assignments forcedAssignments = stableAssignments.force() ? stableAssignments : null;
@@ -940,7 +960,7 @@ public class PartitionReplicaLifecycleManager extends
try {
Entry newEntry = evt.entryEvent().newEntry();
- return handleChangePendingAssignmentEvent(newEntry, evt.revision());
+ return handleChangePendingAssignmentEvent(newEntry, evt.revision(), false);
} finally {
busyLock.leaveBusy();
}
@@ -1141,7 +1161,7 @@ public class PartitionReplicaLifecycleManager extends
);
}
- private CompletableFuture<Void> handleChangePendingAssignmentEvent(Entry pendingAssignmentsEntry, long revision) {
+ private CompletableFuture<Void> handleChangePendingAssignmentEvent(Entry pendingAssignmentsEntry, long revision, boolean isRecovery) {
if (pendingAssignmentsEntry.value() == null || pendingAssignmentsEntry.empty()) {
return nullCompletedFuture();
}
@@ -1177,7 +1197,8 @@ public class PartitionReplicaLifecycleManager extends
zonePartitionId,
stableAssignments,
pendingAssignments,
- revision
+ revision,
+ isRecovery
).thenCompose(v -> {
boolean isLocalNodeInStableOrPending = isNodeInReducedStableOrPendingAssignments(
zonePartitionId,
@@ -1190,6 +1211,16 @@ public class PartitionReplicaLifecycleManager extends
return nullCompletedFuture();
}
+ // A node might exist in stable yet we don't want to start the replica
+ // The case is: nodes A, B and C hold a replication group,
+ // nodes A and B die.
+ // Reset adds C into force pending and user writes data onto C.
+ // Then A and B go back online. In this case
+ // stable = [A, B, C], pending = [C, force] and only C should be started.
+ if (isRecovery && !replicaMgr.isReplicaStarted(zonePartitionId)) {
+ return nullCompletedFuture();
+ }
+
return changePeersOnRebalance(
replicaMgr,
zonePartitionId,
@@ -1205,16 +1236,26 @@ public class PartitionReplicaLifecycleManager extends
ZonePartitionId replicaGrpId,
@Nullable Assignments stableAssignments,
Assignments pendingAssignments,
- long revision
+ long revision,
+ boolean isRecovery
) {
boolean pendingAssignmentsAreForced = pendingAssignments.force();
Set<Assignment> pendingAssignmentsNodes = pendingAssignments.nodes();
// Start a new Raft node and Replica if this node has appeared in the new assignments.
- Assignment localAssignment = localAssignment(pendingAssignments);
-
- boolean shouldStartLocalGroupNode = localAssignment != null
- && (stableAssignments == null || !stableAssignments.nodes().contains(localAssignment));
+ Assignment localAssignmentInPending = localAssignment(pendingAssignments);
+ Assignment localAssignmentInStable = localAssignment(stableAssignments);
+
+ boolean shouldStartLocalGroupNode;
+ if (isRecovery) {
+ // The condition to start the replica is
+ // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+ // This condition covers the left part of the OR expression.
+ // The right part of it is covered in `startLocalPartitionsAndClients`.
+ shouldStartLocalGroupNode = localAssignmentInPending != null;
+ } else {
+ shouldStartLocalGroupNode = localAssignmentInPending != null && localAssignmentInStable == null;
+ }
// This is a set of assignments for nodes that are not the part of stable assignments, i.e. unstable part of the distribution.
// For regular pending assignments we use (old) stable set, so that none of new nodes would be able to propose itself as a leader.
@@ -1249,14 +1290,14 @@ public class PartitionReplicaLifecycleManager extends
localServicesStartFuture = createZonePartitionReplicationNode(
replicaGrpId,
- localAssignment,
+ localAssignmentInPending,
computedStableAssignments,
revision,
zoneDescriptor.partitions(),
isVolatileZone(zoneDescriptor),
false
);
- } else if (pendingAssignmentsAreForced && localAssignment != null) {
+ } else if (pendingAssignmentsAreForced && localAssignmentInPending != null) {
localServicesStartFuture = runAsync(() -> {
inBusyLock(busyLock, () -> replicaMgr.resetPeers(replicaGrpId, fromAssignments(computedStableAssignments.nodes())));
}, ioExecutor);
@@ -1282,6 +1323,21 @@ public class PartitionReplicaLifecycleManager extends
: "The local node is outside of the replication group [inStableOrPending=" + isLocalNodeInStableOrPending
+ ", isLeaseholder=" + isLeaseholder + "].";
+ // A node might exist in stable yet we don't want to start the replica
+ // The case is: nodes A, B and C hold a replication group,
+ // nodes A and B die.
+ // Reset adds C into force pending and user writes data onto C.
+ // Then A and B go back online. In this case
+ // stable = [A, B, C], pending = [C, force] and only C should be started.
+ if (isRecovery && !replicaMgr.isReplicaStarted(replicaGrpId)) {
+ return;
+ }
+
+ assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
+ + ", stable=" + stableAssignments
+ + ", pending=" + pendingAssignments
+ + ", localName=" + localNode().name() + "].";
+
// For forced assignments, we exclude dead stable nodes, and all alive stable nodes are already in pending assignments.
// Union is not required in such a case.
Set<Assignment> newAssignments = pendingAssignmentsAreForced || stableAssignments == null
@@ -1432,17 +1488,7 @@ public class PartitionReplicaLifecycleManager extends
? subtract(stableAssignments.nodes(), reduceAssignments.nodes())
: stableAssignments.nodes();
- if (!isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()))) {
- return false;
- }
-
- assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
- + ", stable=" + stableAssignments
- + ", pending=" + pendingAssignments
- + ", reduce=" + reduceAssignments
- + ", localName=" + localNode().name() + "].";
-
- return true;
+ return isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()));
}
@Nullable
diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java
index 5a8d56c6c05..5afd2824f61 100644
--- a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java
+++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java
@@ -104,6 +104,7 @@ public class ItDisasterRecoveryManagerTest extends ClusterPerTestIntegrationTest
));
}
+ @WithSystemProperty(key = COLOCATION_FEATURE_FLAG, value = "false")
@Test
void testRestartTablePartitions() {
IgniteImpl node = unwrapIgniteImpl(cluster.aliveNode());
diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
index b46bb7815df..35e23ebfe4c 100644
--- a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
+++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
@@ -409,8 +409,6 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
}
@Test
- // TODO https://issues.apache.org/jira/browse/IGNITE-24232
- @WithSystemProperty(key = COLOCATION_FEATURE_FLAG, value = "false")
@ZoneParams(nodes = 5, replicas = 3, partitions = 1)
@MuteFailureManagerLogging
void testManualRebalanceRecovery() throws Exception {