You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sa...@apache.org on 2025/04/30 16:01:27 UTC

(ignite-3) branch main updated: IGNITE-24232 Implement correct node restart behavior when forced pending (#5726)

This is an automated email from the ASF dual-hosted git repository.

sanpwc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git


The following commit(s) were added to refs/heads/main by this push:
     new ccf40c3fdc5 IGNITE-24232 Implement correct node restart behavior when forced pending (#5726)
ccf40c3fdc5 is described below

commit ccf40c3fdc5508fa09b62c9e950e1622f4b0f42f
Author: Cyrill <cy...@gmail.com>
AuthorDate: Wed Apr 30 19:01:21 2025 +0300

    IGNITE-24232 Implement correct node restart behavior when forced pending (#5726)
---
 .../rebalance/ZoneRebalanceUtil.java               |  37 +++++-
 .../PartitionReplicaLifecycleManager.java          | 136 ++++++++++++++-------
 .../disaster/ItDisasterRecoveryManagerTest.java    |   1 +
 .../ItDisasterRecoveryReconfigurationTest.java     |   2 -
 4 files changed, 123 insertions(+), 53 deletions(-)

diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
index abe21910c15..c49519e7fb9 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
@@ -577,9 +577,9 @@ public class ZoneRebalanceUtil {
             int partitionNumber,
             long revision
     ) {
-        Entry entry = metaStorageManager.getLocally(stablePartAssignmentsKey(new ZonePartitionId(zoneId, partitionNumber)), revision);
-
-        return (entry == null || entry.empty() || entry.tombstone()) ? null : Assignments.fromBytes(entry.value()).nodes();
+        Assignments assignments =
+                zoneStableAssignmentsGetLocally(metaStorageManager, new ZonePartitionId(zoneId, partitionNumber), revision);
+        return assignments == null ? null : assignments.nodes();
     }
 
     /**
@@ -599,11 +599,36 @@ public class ZoneRebalanceUtil {
     ) {
         return IntStream.range(0, numberOfPartitions)
                 .mapToObj(p -> {
-                    Entry e = metaStorageManager.getLocally(stablePartAssignmentsKey(new ZonePartitionId(zoneId, p)), revision);
+                    Assignments assignments =
+                            zoneStableAssignmentsGetLocally(metaStorageManager, new ZonePartitionId(zoneId, p), revision);
+
+                    assert assignments != null : "No assignments found for " + new ZonePartitionId(zoneId, p);
+
+                    return assignments;
+                })
+                .collect(toList());
+    }
 
-                    assert e != null && !e.empty() && !e.tombstone() : e;
+    /**
+     * Returns zone pending assignments for all zone partitions from meta storage locally. Assignments must be present.
+     *
+     * @param metaStorageManager Meta storage manager.
+     * @param zoneId Zone id.
+     * @param numberOfPartitions Number of partitions.
+     * @param revision Revision.
+     * @return Future with zone assignments as a value.
+     */
+    public static List<@Nullable Assignments> zonePendingAssignmentsGetLocally(
+            MetaStorageManager metaStorageManager,
+            int zoneId,
+            int numberOfPartitions,
+            long revision
+    ) {
+        return IntStream.range(0, numberOfPartitions)
+                .mapToObj(p -> {
+                    Entry e = metaStorageManager.getLocally(pendingPartAssignmentsQueueKey(new ZonePartitionId(zoneId, p)), revision);
 
-                    return Assignments.fromBytes(e.value());
+                    return e != null && !e.empty() && !e.tombstone() ? AssignmentsQueue.fromBytes(e.value()).poll() : null;
                 })
                 .collect(toList());
     }
diff --git a/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java b/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java
index afa2cc361a1..870fcd6c8fd 100644
--- a/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java
+++ b/modules/partition-replicator/src/main/java/org/apache/ignite/internal/partition/replicator/PartitionReplicaLifecycleManager.java
@@ -38,6 +38,7 @@ import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalan
 import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.stablePartAssignmentsKey;
 import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zoneAssignmentsGetLocally;
 import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zonePartitionAssignmentsGetLocally;
+import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zonePendingAssignmentsGetLocally;
 import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.zoneStableAssignmentsGetLocally;
 import static org.apache.ignite.internal.hlc.HybridTimestamp.LOGICAL_TIME_BITS_SIZE;
 import static org.apache.ignite.internal.hlc.HybridTimestamp.hybridTimestamp;
@@ -462,7 +463,7 @@ public class PartitionReplicaLifecycleManager extends
         return handleAssignmentsOnRecovery(
                 new ByteArray(PENDING_ASSIGNMENTS_QUEUE_PREFIX_BYTES),
                 recoveryRevision,
-                this::handleChangePendingAssignmentEvent,
+                (pendingAssignmentsEntry, revision) -> handleChangePendingAssignmentEvent(pendingAssignmentsEntry, revision, true),
                 "pending"
         );
     }
@@ -525,9 +526,10 @@ public class PartitionReplicaLifecycleManager extends
             return getOrCreateAssignments(zoneDescriptor, causalityToken, catalogVersion)
                     .thenCompose(assignments -> writeZoneAssignmentsToMetastore(zoneId, assignments))
                     .thenCompose(
-                            assignments -> createZoneReplicationNodes(
+                            stableAssignments -> createZoneReplicationNodes(
                                     zoneId,
-                                    assignments,
+                                    stableAssignments,
+                                    zonePendingAssignmentsGetLocally(metaStorageMgr, zoneId, zoneDescriptor.partitions(), causalityToken),
                                     causalityToken,
                                     catalogVersion,
                                     zoneDescriptor.partitions(),
@@ -539,33 +541,55 @@ public class PartitionReplicaLifecycleManager extends
 
     private CompletableFuture<Void> createZoneReplicationNodes(
             int zoneId,
-            List<Assignments> assignments,
+            List<Assignments> stableAssignmentsForZone,
+            List<@Nullable Assignments> pendingAssignmentsForZone,
             long causalityToken,
             int catalogVersion,
             int partitionCount,
             boolean onNodeRecovery
     ) {
-        assert assignments != null : IgniteStringFormatter.format("Zone has empty assignments [id={}].", zoneId);
+        assert stableAssignmentsForZone != null : IgniteStringFormatter.format("Zone has empty assignments [id={}].", zoneId);
 
         Supplier<CompletableFuture<Void>> createZoneReplicationNodes = () -> inBusyLockAsync(busyLock, () -> {
-            var partitionsStartFutures = new CompletableFuture<?>[assignments.size()];
+            var partitionsStartFutures = new CompletableFuture<?>[stableAssignmentsForZone.size()];
 
-            for (int partId = 0; partId < assignments.size(); partId++) {
-                Assignments zoneAssignment = assignments.get(partId);
+            for (int partId = 0; partId < stableAssignmentsForZone.size(); partId++) {
+                Assignments stableAssignments = stableAssignmentsForZone.get(partId);
 
-                Assignment localAssignment = localAssignment(zoneAssignment);
+                Assignments pendingAssignments = pendingAssignmentsForZone.get(partId);
 
-                var zonePartitionId = new ZonePartitionId(zoneId, partId);
+                Assignment localAssignment = localAssignment(stableAssignments);
+
+                boolean shouldStartPartition;
+
+                if (onNodeRecovery) {
+                    // The condition to start the replica is
+                    // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+                    // However we check only the right part of this condition here
+                    // since after `startTables` we have a call to `processAssignmentsOnRecovery`,
+                    // which executes pending assignments update and will start required partitions there.
+                    shouldStartPartition = localAssignment != null
+                            && (pendingAssignments == null || !pendingAssignments.force());
+                } else {
+                    shouldStartPartition = localAssignment != null;
+                }
+
+                if (shouldStartPartition) {
+                    var zonePartitionId = new ZonePartitionId(zoneId, partId);
+
+                    partitionsStartFutures[partId] = createZonePartitionReplicationNode(
+                            zonePartitionId,
+                            localAssignment,
+                            stableAssignments,
+                            causalityToken,
+                            partitionCount,
+                            isVolatileZoneForCatalogVersion(zoneId, catalogVersion),
+                            !onNodeRecovery
+                    );
+                } else {
+                    partitionsStartFutures[partId] = nullCompletedFuture();
+                }
 
-                partitionsStartFutures[partId] = createZonePartitionReplicationNode(
-                        zonePartitionId,
-                        localAssignment,
-                        zoneAssignment,
-                        causalityToken,
-                        partitionCount,
-                        isVolatileZoneForCatalogVersion(zoneId, catalogVersion),
-                        !onNodeRecovery
-                );
             }
 
             return allOf(partitionsStartFutures);
@@ -620,17 +644,13 @@ public class PartitionReplicaLifecycleManager extends
      */
     private CompletableFuture<?> createZonePartitionReplicationNode(
             ZonePartitionId zonePartitionId,
-            @Nullable Assignment localAssignment,
+            Assignment localAssignment,
             Assignments stableAssignments,
             long revision,
             int partitionCount,
             boolean isVolatileZone,
             boolean holdingZoneWriteLock
     ) {
-        if (localAssignment == null) {
-            return nullCompletedFuture();
-        }
-
         // TODO: https://issues.apache.org/jira/browse/IGNITE-22522 We need to integrate PartitionReplicatorNodeRecovery logic here when
         //  we in the recovery phase.
         Assignments forcedAssignments = stableAssignments.force() ? stableAssignments : null;
@@ -940,7 +960,7 @@ public class PartitionReplicaLifecycleManager extends
             try {
                 Entry newEntry = evt.entryEvent().newEntry();
 
-                return handleChangePendingAssignmentEvent(newEntry, evt.revision());
+                return handleChangePendingAssignmentEvent(newEntry, evt.revision(), false);
             } finally {
                 busyLock.leaveBusy();
             }
@@ -1141,7 +1161,7 @@ public class PartitionReplicaLifecycleManager extends
         );
     }
 
-    private CompletableFuture<Void> handleChangePendingAssignmentEvent(Entry pendingAssignmentsEntry, long revision) {
+    private CompletableFuture<Void> handleChangePendingAssignmentEvent(Entry pendingAssignmentsEntry, long revision, boolean isRecovery) {
         if (pendingAssignmentsEntry.value() == null || pendingAssignmentsEntry.empty()) {
             return nullCompletedFuture();
         }
@@ -1177,7 +1197,8 @@ public class PartitionReplicaLifecycleManager extends
                     zonePartitionId,
                     stableAssignments,
                     pendingAssignments,
-                    revision
+                    revision,
+                    isRecovery
             ).thenCompose(v -> {
                 boolean isLocalNodeInStableOrPending = isNodeInReducedStableOrPendingAssignments(
                         zonePartitionId,
@@ -1190,6 +1211,16 @@ public class PartitionReplicaLifecycleManager extends
                     return nullCompletedFuture();
                 }
 
+                // A node might exist in stable yet we don't want to start the replica
+                // The case is: nodes A, B and C hold a replication group,
+                // nodes A and B die.
+                // Reset adds C into force pending and user writes data onto C.
+                // Then A and B go back online. In this case
+                // stable = [A, B, C], pending = [C, force] and only C should be started.
+                if (isRecovery && !replicaMgr.isReplicaStarted(zonePartitionId)) {
+                    return nullCompletedFuture();
+                }
+
                 return changePeersOnRebalance(
                         replicaMgr,
                         zonePartitionId,
@@ -1205,16 +1236,26 @@ public class PartitionReplicaLifecycleManager extends
             ZonePartitionId replicaGrpId,
             @Nullable Assignments stableAssignments,
             Assignments pendingAssignments,
-            long revision
+            long revision,
+            boolean isRecovery
     ) {
         boolean pendingAssignmentsAreForced = pendingAssignments.force();
         Set<Assignment> pendingAssignmentsNodes = pendingAssignments.nodes();
 
         // Start a new Raft node and Replica if this node has appeared in the new assignments.
-        Assignment localAssignment = localAssignment(pendingAssignments);
-
-        boolean shouldStartLocalGroupNode = localAssignment != null
-                && (stableAssignments == null || !stableAssignments.nodes().contains(localAssignment));
+        Assignment localAssignmentInPending = localAssignment(pendingAssignments);
+        Assignment localAssignmentInStable = localAssignment(stableAssignments);
+
+        boolean shouldStartLocalGroupNode;
+        if (isRecovery) {
+            // The condition to start the replica is
+            // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+            // This condition covers the left part of the OR expression.
+            // The right part of it is covered in `startLocalPartitionsAndClients`.
+            shouldStartLocalGroupNode = localAssignmentInPending != null;
+        } else {
+            shouldStartLocalGroupNode = localAssignmentInPending != null && localAssignmentInStable == null;
+        }
 
         // This is a set of assignments for nodes that are not the part of stable assignments, i.e. unstable part of the distribution.
         // For regular pending assignments we use (old) stable set, so that none of new nodes would be able to propose itself as a leader.
@@ -1249,14 +1290,14 @@ public class PartitionReplicaLifecycleManager extends
 
             localServicesStartFuture = createZonePartitionReplicationNode(
                     replicaGrpId,
-                    localAssignment,
+                    localAssignmentInPending,
                     computedStableAssignments,
                     revision,
                     zoneDescriptor.partitions(),
                     isVolatileZone(zoneDescriptor),
                     false
             );
-        } else if (pendingAssignmentsAreForced && localAssignment != null) {
+        } else if (pendingAssignmentsAreForced && localAssignmentInPending != null) {
             localServicesStartFuture = runAsync(() -> {
                 inBusyLock(busyLock, () -> replicaMgr.resetPeers(replicaGrpId, fromAssignments(computedStableAssignments.nodes())));
             }, ioExecutor);
@@ -1282,6 +1323,21 @@ public class PartitionReplicaLifecycleManager extends
                             : "The local node is outside of the replication group [inStableOrPending=" + isLocalNodeInStableOrPending
                             + ", isLeaseholder=" + isLeaseholder + "].";
 
+                    // A node might exist in stable yet we don't want to start the replica
+                    // The case is: nodes A, B and C hold a replication group,
+                    // nodes A and B die.
+                    // Reset adds C into force pending and user writes data onto C.
+                    // Then A and B go back online. In this case
+                    // stable = [A, B, C], pending = [C, force] and only C should be started.
+                    if (isRecovery && !replicaMgr.isReplicaStarted(replicaGrpId)) {
+                        return;
+                    }
+
+                    assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
+                            + ", stable=" + stableAssignments
+                            + ", pending=" + pendingAssignments
+                            + ", localName=" + localNode().name() + "].";
+
                     // For forced assignments, we exclude dead stable nodes, and all alive stable nodes are already in pending assignments.
                     // Union is not required in such a case.
                     Set<Assignment> newAssignments = pendingAssignmentsAreForced || stableAssignments == null
@@ -1432,17 +1488,7 @@ public class PartitionReplicaLifecycleManager extends
                 ? subtract(stableAssignments.nodes(), reduceAssignments.nodes())
                 : stableAssignments.nodes();
 
-        if (!isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()))) {
-            return false;
-        }
-
-        assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
-                + ", stable=" + stableAssignments
-                + ", pending=" + pendingAssignments
-                + ", reduce=" + reduceAssignments
-                + ", localName=" + localNode().name() + "].";
-
-        return true;
+        return isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()));
     }
 
     @Nullable
diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java
index 5a8d56c6c05..5afd2824f61 100644
--- a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java
+++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryManagerTest.java
@@ -104,6 +104,7 @@ public class ItDisasterRecoveryManagerTest extends ClusterPerTestIntegrationTest
         ));
     }
 
+    @WithSystemProperty(key = COLOCATION_FEATURE_FLAG, value = "false")
     @Test
     void testRestartTablePartitions() {
         IgniteImpl node = unwrapIgniteImpl(cluster.aliveNode());
diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
index b46bb7815df..35e23ebfe4c 100644
--- a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
+++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
@@ -409,8 +409,6 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     }
 
     @Test
-    // TODO https://issues.apache.org/jira/browse/IGNITE-24232
-    @WithSystemProperty(key = COLOCATION_FEATURE_FLAG, value = "false")
     @ZoneParams(nodes = 5, replicas = 3, partitions = 1)
     @MuteFailureManagerLogging
     void testManualRebalanceRecovery() throws Exception {