You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sk...@apache.org on 2024/12/16 17:55:42 UTC

(ignite-3) branch main updated: IGNITE-23780 Node restart behaviour when forced pending (#4847)

This is an automated email from the ASF dual-hosted git repository.

sk0x50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git


The following commit(s) were added to refs/heads/main by this push:
     new 6a4fafe855 IGNITE-23780 Node restart behaviour when forced pending (#4847)
6a4fafe855 is described below

commit 6a4fafe855859eaec5ca83280c6fdcb076cecb9d
Author: Cyrill <cy...@gmail.com>
AuthorDate: Mon Dec 16 20:55:35 2024 +0300

    IGNITE-23780 Node restart behaviour when forced pending (#4847)
    
    Co-authored-by: Kirill Sizov <si...@gmail.com>
---
 .../distributionzones/DistributionZoneManager.java |   1 +
 .../distributionzones/rebalance/RebalanceUtil.java |  24 +++
 .../org/apache/ignite/internal/app/IgniteImpl.java |   5 +
 .../ItDisasterRecoveryReconfigurationTest.java     | 200 +++++++++++++++++--
 .../internal/table/distributed/TableManager.java   | 216 +++++++++++++--------
 5 files changed, 344 insertions(+), 102 deletions(-)

diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
index 048a49c6bf..86408d3820 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
@@ -181,6 +181,7 @@ public class DistributionZoneManager extends
 
     /**
      * The logical topology mapped to the MS revision.
+     * TODO: https://issues.apache.org/jira/browse/IGNITE-23561 get rid of this map (or properly clean up it).
      */
     private final ConcurrentSkipListMap<Long, Set<NodeWithAttributes>> logicalTopologyByRevision = new ConcurrentSkipListMap<>();
 
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
index 8434986f4f..fe08e9585b 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
@@ -636,4 +636,28 @@ public class RebalanceUtil {
                 })
                 .collect(toList());
     }
+
+    /**
+     * Returns table pending assignments for all table partitions from meta storage locally.
+     *
+     * @param metaStorageManager Meta storage manager.
+     * @param tableId Table id.
+     * @param numberOfPartitions Number of partitions.
+     * @param revision Revision.
+     * @return Future with table assignments as a value.
+     */
+    public static List<Assignments> tablePendingAssignmentsGetLocally(
+            MetaStorageManager metaStorageManager,
+            int tableId,
+            int numberOfPartitions,
+            long revision
+    ) {
+        return IntStream.range(0, numberOfPartitions)
+                .mapToObj(p -> {
+                    Entry e = metaStorageManager.getLocally(pendingPartAssignmentsKey(new TablePartitionId(tableId, p)), revision);
+
+                    return e != null ? Assignments.fromBytes(e.value()) : null;
+                })
+                .collect(toList());
+    }
 }
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
index fe033dcfe0..735f8913a6 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
@@ -1589,6 +1589,11 @@ public class IgniteImpl implements Ignite {
         return metricManager;
     }
 
+    @TestOnly
+    public TableManager distributedTableManager() {
+        return distributedTblMgr;
+    }
+
     /** {@inheritDoc} */
     @Override
     public IgniteTransactions transactions() {
diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
index 6b2a776a81..f223d9652d 100644
--- a/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
+++ b/modules/table/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java
@@ -18,12 +18,12 @@
 package org.apache.ignite.internal.disaster;
 
 import static java.lang.String.format;
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptySet;
 import static java.util.Map.of;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl;
 import static org.apache.ignite.internal.TestWrappers.unwrapTableImpl;
 import static org.apache.ignite.internal.TestWrappers.unwrapTableManager;
 import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE;
@@ -99,6 +99,8 @@ import org.apache.ignite.internal.table.distributed.disaster.LocalPartitionState
 import org.apache.ignite.lang.IgniteException;
 import org.apache.ignite.raft.jraft.RaftGroupService;
 import org.apache.ignite.raft.jraft.Status;
+import org.apache.ignite.raft.jraft.core.NodeImpl;
+import org.apache.ignite.raft.jraft.entity.LogId;
 import org.apache.ignite.raft.jraft.error.RaftError;
 import org.apache.ignite.raft.jraft.rpc.RpcRequests.AppendEntriesRequest;
 import org.apache.ignite.raft.jraft.rpc.WriteActionRequest;
@@ -150,7 +152,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void setUp(TestInfo testInfo) throws Exception {
         Method testMethod = testInfo.getTestMethod().orElseThrow();
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
 
         zoneName = "ZONE_" + testMethod.getName().toUpperCase();
 
@@ -185,7 +187,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testInsertFailsIfMajorityIsLost() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         Table table = node0.tables().table(TABLE_NAME);
 
         awaitPrimaryReplica(node0, partId);
@@ -232,7 +234,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testManualRebalanceIfMajorityIsLost() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         Table table = node0.tables().table(TABLE_NAME);
 
         awaitPrimaryReplica(node0, partId);
@@ -270,7 +272,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
         int fixingPartId = 1;
         int anotherPartId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         Table table = node0.tables().table(TABLE_NAME);
 
         awaitPrimaryReplica(node0, anotherPartId);
@@ -318,7 +320,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testManualRebalanceIfPartitionIsLost() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         Table table = node0.tables().table(TABLE_NAME);
 
         awaitPrimaryReplica(node0, partId);
@@ -345,6 +347,149 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
         assertThat(errors, is(empty()));
     }
 
+    @Test
+    @ZoneParams(nodes = 5, replicas = 3, partitions = 1)
+    void testManualRebalanceRecovery() throws Exception {
+        int partId = 0;
+        // Disable scale down to avoid unwanted rebalance.
+        executeSql(format("ALTER ZONE %s SET data_nodes_auto_adjust_scale_down=%d", zoneName, INFINITE_TIMER_VALUE));
+
+        IgniteImpl node0 = igniteImpl(0);
+        int catalogVersion = node0.catalogManager().latestCatalogVersion();
+        long timestamp = node0.catalogManager().catalog(catalogVersion).time();
+        Table table = node0.tables().table(TABLE_NAME);
+
+        awaitPrimaryReplica(node0, partId);
+
+        Assignments allAssignments = Assignments.of(timestamp,
+                Assignment.forPeer(node(0).name()),
+                Assignment.forPeer(node(1).name()),
+                Assignment.forPeer(node(2).name())
+        );
+
+        // The partition is replicated to 3 nodes.
+        assertRealAssignments(node0, partId, 0, 1, 2);
+
+        List<Throwable> errors = insertValues(table, partId, 0);
+        assertThat(errors, is(empty()));
+
+        // Make sure node 0 has the same data as node 1 and 2.
+        assertTrue(
+                waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(1, partId)), SECONDS.toMillis(20)),
+                () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 1 log index= " + getRaftLogIndex(1, partId)
+        );
+        assertTrue(
+                waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(2, partId)), SECONDS.toMillis(20)),
+                () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 2 log index= " + getRaftLogIndex(2, partId)
+        );
+
+        // Stop 1 and 2, only 0 survived.
+        stopNodesInParallel(1, 2);
+
+        Assignments assignment0 = Assignments.of(timestamp,
+                Assignment.forPeer(node(0).name())
+        );
+
+        // Blocking stable switch to the first phase or reset,
+        // so that we'll have force pending assignments unexecuted.
+        blockMessage((nodeName, msg) -> stableKeySwitchMessage(msg, partId, assignment0));
+
+        // Init reset:
+        // pending = [0, force]
+        // planned = [0, 3, 4]
+        CompletableFuture<?> updateFuture = node0.disasterRecoveryManager().resetAllPartitions(
+                zoneName,
+                QUALIFIED_TABLE_NAME,
+                true
+        );
+
+        assertThat(updateFuture, willCompleteSuccessfully());
+
+        awaitPrimaryReplica(node0, partId);
+
+        // Since stable switch to 0 is blocked, data is stored on 0 node only.
+        assertRealAssignments(node0, partId, 0);
+
+        // And stable is in its initial state.
+        assertStableAssignments(node0, partId, allAssignments);
+
+        // Insert new data on the node 0.
+        errors = insertValues(table, partId, 10);
+        assertThat(errors, is(empty()));
+
+        // Start nodes 1 and 2 back, but they should not start their replicas.
+        startNode(1);
+        startNode(2);
+
+        // Make sure recovery has finished.
+        assertThat(igniteImpl(1).distributedTableManager().recoveryFuture(), willCompleteSuccessfully());
+        assertThat(igniteImpl(2).distributedTableManager().recoveryFuture(), willCompleteSuccessfully());
+
+        // Make sure 1 and 2 did not start.
+        assertRealAssignments(node0, partId, 0);
+    }
+
+    @Test
+    @ZoneParams(nodes = 4, replicas = 3, partitions = 1)
+    void testManualRebalanceRecoveryNoPending() throws Exception {
+        int partId = 0;
+
+        IgniteImpl node0 = igniteImpl(0);
+        int catalogVersion = node0.catalogManager().latestCatalogVersion();
+        long timestamp = node0.catalogManager().catalog(catalogVersion).time();
+        Table table = node0.tables().table(TABLE_NAME);
+
+        awaitPrimaryReplica(node0, partId);
+
+        Assignments allAssignments = Assignments.of(timestamp,
+                Assignment.forPeer(node(0).name()),
+                Assignment.forPeer(node(1).name()),
+                Assignment.forPeer(node(3).name())
+        );
+
+        // The partition is replicated to 3 nodes.
+        assertRealAssignments(node0, partId, 0, 1, 3);
+
+        List<Throwable> errors = insertValues(table, partId, 0);
+        assertThat(errors, is(empty()));
+
+        // Make sure node 0 has the same data as node 1 and 2.
+        assertTrue(
+                waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(1, partId)), SECONDS.toMillis(20)),
+                () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 1 log index= " + getRaftLogIndex(1, partId)
+        );
+        assertTrue(
+                waitForCondition(() -> getRaftLogIndex(0, partId).equals(getRaftLogIndex(3, partId)), SECONDS.toMillis(20)),
+                () -> "Node 0 log index = " + getRaftLogIndex(0, partId) + " node 2 log index= " + getRaftLogIndex(3, partId)
+        );
+
+        Assignments assignmentPending = Assignments.of(timestamp,
+                Assignment.forPeer(node(0).name()),
+                Assignment.forPeer(node(1).name()),
+                Assignment.forPeer(node(2).name())
+        );
+
+        // Blocking stable switch to the first phase or reset,
+        // so that we'll have force pending assignments unexecuted.
+        blockMessage((nodeName, msg) -> stableKeySwitchMessage(msg, partId, assignmentPending));
+
+        // Stop 3. Nodes 0 and 1 survived.
+        stopNode(3);
+
+        waitForScale(node0, 3);
+
+        assertRealAssignments(node0, partId, 0, 1, 2);
+
+        assertPendingAssignments(node0, partId, assignmentPending);
+        assertStableAssignments(node0, partId, allAssignments);
+
+        // On start we check `pending.contains(node) || (stable.contains(node) && !pending.isForce())` condition.
+        // The node is in stable, not in pending and pending is not forced => start it.
+        startNode(3);
+
+        assertRealAssignments(node0, partId, 0, 1, 2, 3);
+    }
+
     /**
      * Tests a scenario where all stable nodes are lost, yet we have data on one of pending nodes and perform reset partition operation. In
      * this case we should use that pending node as a source of data for recovery.
@@ -363,7 +508,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     public void testIncompleteRebalanceAfterResetPartitions() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
 
         int catalogVersion = node0.catalogManager().latestCatalogVersion();
         long timestamp = node0.catalogManager().catalog(catalogVersion).time();
@@ -379,7 +524,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
         // Second snapshot causes log truncation.
         triggerRaftSnapshot(1, partId);
 
-        unwrapIgniteImpl(node(1)).dropMessages((nodeName, msg) -> {
+        igniteImpl(1).dropMessages((nodeName, msg) -> {
             Ignite node = nullableNode(3);
 
             return node != null && node.name().equals(nodeName) && msg instanceof SnapshotMvDataResponse;
@@ -454,7 +599,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testAutomaticRebalanceIfMajorityIsLost() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         Table table = node0.tables().table(TABLE_NAME);
 
         awaitPrimaryReplica(node0, partId);
@@ -495,9 +640,9 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testAutomaticRebalanceIfPartitionIsLost() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
 
-        IgniteImpl node1 = unwrapIgniteImpl(cluster.node(1));
+        IgniteImpl node1 = igniteImpl(1);
 
         executeSql(format("ALTER ZONE %s SET data_nodes_auto_adjust_scale_down=%d", zoneName, INFINITE_TIMER_VALUE));
 
@@ -540,7 +685,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     public void testIncompleteRebalanceBeforeAutomaticResetPartitions() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
 
         int catalogVersion = node0.catalogManager().latestCatalogVersion();
         long timestamp = node0.catalogManager().catalog(catalogVersion).time();
@@ -610,7 +755,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testThoPhaseResetMaxLogIndex() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         int catalogVersion = node0.catalogManager().latestCatalogVersion();
         long timestamp = node0.catalogManager().catalog(catalogVersion).time();
         Table table = node0.tables().table(TABLE_NAME);
@@ -748,7 +893,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testThoPhaseResetEqualLogIndex() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         int catalogVersion = node0.catalogManager().latestCatalogVersion();
         long timestamp = node0.catalogManager().catalog(catalogVersion).time();
         Table table = node0.tables().table(TABLE_NAME);
@@ -870,7 +1015,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     void testTwoPhaseResetOnEmptyNodes() throws Exception {
         int partId = 0;
 
-        IgniteImpl node0 = unwrapIgniteImpl(cluster.node(0));
+        IgniteImpl node0 = igniteImpl(0);
         int catalogVersion = node0.catalogManager().latestCatalogVersion();
         long timestamp = node0.catalogManager().catalog(catalogVersion).time();
 
@@ -1007,7 +1152,7 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
     }
 
     private String findLeader(int nodeIdx, int partId) {
-        IgniteImpl node = unwrapIgniteImpl(node(nodeIdx));
+        IgniteImpl node = igniteImpl(nodeIdx);
 
         var raftNodeId = new RaftNodeId(new TablePartitionId(tableId, partId), new Peer(node.name()));
         var jraftServer = (JraftServerImpl) node.raftManager().server();
@@ -1017,9 +1162,24 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
         return raftGroupService.getRaftNode().getLeaderId().getConsistentId();
     }
 
+    private NodeImpl getRaftNode(int nodeIdx, int partId) {
+        IgniteImpl node = igniteImpl(nodeIdx);
+
+        var raftNodeId = new RaftNodeId(new TablePartitionId(tableId, partId), new Peer(node.name()));
+        var jraftServer = (JraftServerImpl) node.raftManager().server();
+
+        RaftGroupService raftGroupService = jraftServer.raftGroupService(raftNodeId);
+        assertNotNull(raftGroupService);
+
+        return (NodeImpl) raftGroupService.getRaftNode();
+    }
+
+    private LogId getRaftLogIndex(int nodeIdx, int partId) {
+        return getRaftNode(nodeIdx, partId).lastLogIndexAndTerm();
+    }
+
     private void triggerRaftSnapshot(int nodeIdx, int partId) throws InterruptedException, ExecutionException {
-        //noinspection resource
-        IgniteImpl node = unwrapIgniteImpl(node(nodeIdx));
+        IgniteImpl node = igniteImpl(nodeIdx);
 
         var raftNodeId = new RaftNodeId(new TablePartitionId(tableId, partId), new Peer(node.name()));
         var jraftServer = (JraftServerImpl) node.raftManager().server();
@@ -1158,6 +1318,10 @@ public class ItDisasterRecoveryReconfigurationTest extends ClusterPerTestIntegra
 
         LocalPartitionStateByNode partitionStates = partitionStatesFut.join().get(new TablePartitionId(tableId, partId));
 
+        if (partitionStates == null) {
+            return emptyList();
+        }
+
         return partitionStates.keySet()
                 .stream()
                 .map(cluster::nodeIndex)
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index 21d5968f32..4c054ffba4 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -39,6 +39,7 @@ import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUt
 import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.stablePartAssignmentsKey;
 import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.subtract;
 import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.tableAssignmentsGetLocally;
+import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.tablePendingAssignmentsGetLocally;
 import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.union;
 import static org.apache.ignite.internal.event.EventListener.fromConsumer;
 import static org.apache.ignite.internal.hlc.HybridTimestamp.LOGICAL_TIME_BITS_SIZE;
@@ -294,7 +295,8 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
     /**
      * Versioned value for tracking RAFT groups initialization and starting completion.
      *
-     * <p>Only explicitly updated in {@link #startLocalPartitionsAndClients(CompletableFuture, TableImpl, int, boolean, long)}.
+     * <p>Only explicitly updated in
+     * {@link #startLocalPartitionsAndClients(CompletableFuture, List, TableImpl, boolean, long)}.
      *
      * <p>Completed strictly after {@link #localPartitionsVv}.
      */
@@ -425,6 +427,8 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
 
     private final Map<Integer, Set<TableImpl>> tablesPerZone = new ConcurrentHashMap<>();
 
+    private final CompletableFuture<Void> recoveryFuture = new CompletableFuture<>();
+
     /**
      * Creates a new table manager.
      *
@@ -658,6 +662,11 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
         });
     }
 
+    @TestOnly
+    public CompletableFuture<Void> recoveryFuture() {
+        return recoveryFuture;
+    }
+
     private CompletableFuture<Void> waitForMetadataCompleteness(long ts) {
         return executorInclinedSchemaSyncService.waitForMetadataCompleteness(HybridTimestamp.hybridTimestamp(ts));
     }
@@ -738,7 +747,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
             CatalogTableDescriptor tableDescriptor,
             boolean onNodeRecovery
     ) {
-        TableImpl table =  createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
+        TableImpl table = createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
 
         int tableId = tableDescriptor.id();
 
@@ -896,7 +905,13 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                 recoveryRevision,
                 (entry, rev) -> handleChangePendingAssignmentEvent(entry, rev, true),
                 "pending"
-        ));
+        )).whenComplete((v, th) -> {
+            if (th != null) {
+                recoveryFuture.completeExceptionally(th);
+            } else {
+                recoveryFuture.complete(null);
+            }
+        });
     }
 
     private CompletableFuture<Void> handleAssignmentsOnRecovery(
@@ -1065,53 +1080,71 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
     /**
      * Updates or creates partition raft groups and storages.
      *
-     * @param assignmentsFuture Table assignments.
+     * @param stableAssignmentsFuture Table assignments.
      * @param table Initialized table entity.
-     * @param zoneId Zone id.
      * @param isRecovery {@code true} if the node is being started up.
      * @param assignmentsTimestamp Assignments timestamp.
      * @return future, which will be completed when the partitions creations done.
      */
     private CompletableFuture<Void> startLocalPartitionsAndClients(
-            CompletableFuture<List<Assignments>> assignmentsFuture,
+            CompletableFuture<List<Assignments>> stableAssignmentsFuture,
+            List<@Nullable Assignments> pendingAssignmentsForPartitions,
             TableImpl table,
-            int zoneId,
             boolean isRecovery,
             long assignmentsTimestamp
     ) {
         int tableId = table.tableId();
 
         // Create new raft nodes according to new assignments.
-        return assignmentsFuture.thenCompose(tableAssignments -> {
+        return stableAssignmentsFuture.thenCompose(stableAssignmentsForPartitions -> {
             // Empty assignments might be a valid case if tables are created from within cluster init HOCON
             // configuration, which is not supported now.
-            assert tableAssignments != null : IgniteStringFormatter.format("Table [id={}] has empty assignments.", tableId);
+            assert stableAssignmentsForPartitions != null
+                    : IgniteStringFormatter.format("Table [id={}] has empty assignments.", tableId);
 
-            int partitions = tableAssignments.size();
+            int partitions = stableAssignmentsForPartitions.size();
 
             var futures = new CompletableFuture<?>[partitions];
 
             for (int i = 0; i < partitions; i++) {
                 int partId = i;
 
-                Assignments assignments = tableAssignments.get(i);
+                Assignments stableAssignments = stableAssignmentsForPartitions.get(i);
 
-                CompletableFuture<?> future = startPartitionAndStartClient(
-                        table,
-                        partId,
-                        localMemberAssignment(assignments),
-                        assignments,
-                        zoneId,
-                        isRecovery,
-                        assignmentsTimestamp
-                )
-                        .whenComplete((res, ex) -> {
-                            if (ex != null) {
-                                LOG.warn("Unable to update raft groups on the node [tableId={}, partitionId={}]", ex, tableId, partId);
-                            }
-                        });
+                Assignments pendingAssignments = pendingAssignmentsForPartitions.get(i);
+
+                Assignment localMemberAssignmentInStable = localMemberAssignment(stableAssignments);
+
+                boolean shouldStartPartition;
 
-                futures[i] = future;
+                if (isRecovery) {
+                    // The condition to start the replica is
+                    // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+                    // However we check only the right part of this condition here
+                    // since after `startTables` we have a call to `processAssignmentsOnRecovery`,
+                    // which executes pending assignments update and will start required partitions there.
+                    shouldStartPartition = localMemberAssignmentInStable != null
+                            && (pendingAssignments == null || !pendingAssignments.force());
+                } else {
+                    shouldStartPartition = localMemberAssignmentInStable != null;
+                }
+
+                if (shouldStartPartition) {
+                    futures[i] = startPartitionAndStartClient(
+                            table,
+                            partId,
+                            localMemberAssignmentInStable,
+                            stableAssignments,
+                            isRecovery,
+                            assignmentsTimestamp
+                    ).whenComplete((res, ex) -> {
+                        if (ex != null) {
+                            LOG.warn("Unable to update raft groups on the node [tableId={}, partitionId={}]", ex, tableId, partId);
+                        }
+                    });
+                } else {
+                    futures[i] = nullCompletedFuture();
+                }
             }
 
             return allOf(futures);
@@ -1121,17 +1154,11 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
     private CompletableFuture<Void> startPartitionAndStartClient(
             TableImpl table,
             int partId,
-            @Nullable Assignment localMemberAssignment,
+            Assignment localMemberAssignment,
             Assignments stableAssignments,
-            int zoneId,
             boolean isRecovery,
             long assignmentsTimestamp
     ) {
-        if (localMemberAssignment == null) {
-            // (0) in case if node not in the assignments
-            return nullCompletedFuture();
-        }
-
         int tableId = table.tableId();
 
         var internalTbl = (InternalTableImpl) table.internalTable();
@@ -1244,10 +1271,10 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
     }
 
     @Nullable
-    private Assignment localMemberAssignment(Assignments assignments) {
+    private Assignment localMemberAssignment(@Nullable Assignments assignments) {
         Assignment localMemberAssignment = Assignment.forPeer(localNode().name());
 
-        return assignments.nodes().contains(localMemberAssignment) ? localMemberAssignment : null;
+        return assignments != null && assignments.nodes().contains(localMemberAssignment) ? localMemberAssignment : null;
     }
 
     private PartitionMover createPartitionMover(TablePartitionId replicaGrpId) {
@@ -1570,15 +1597,19 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
             // Retrieve descriptor during synchronous call, before the previous catalog version could be concurrently compacted.
             CatalogZoneDescriptor zoneDescriptor = getZoneDescriptor(tableDescriptor, catalogVersion);
 
-            CompletableFuture<List<Assignments>> assignmentsFuture = getOrCreateAssignments(
+            // Returns stable assignments.
+            CompletableFuture<List<Assignments>> stableAssignmentsFuture = getOrCreateAssignments(
                     tableDescriptor,
                     zoneDescriptor,
                     causalityToken,
                     catalogVersion
             );
 
-            CompletableFuture<List<Assignments>> assignmentsFutureAfterInvoke =
-                    writeTableAssignmentsToMetastore(tableId, assignmentsFuture);
+            List<Assignments> pendingAssignments =
+                    tablePendingAssignmentsGetLocally(metaStorageMgr, tableId, zoneDescriptor.partitions(), causalityToken);
+
+            CompletableFuture<List<Assignments>> stableAssignmentsFutureAfterInvoke =
+                    writeTableAssignmentsToMetastore(tableId, stableAssignmentsFuture);
 
             Catalog catalog = catalogService.catalog(catalogVersion);
 
@@ -1586,7 +1617,8 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                     causalityToken,
                     tableDescriptor,
                     zoneDescriptor,
-                    assignmentsFutureAfterInvoke,
+                    stableAssignmentsFutureAfterInvoke,
+                    pendingAssignments,
                     onNodeRecovery,
                     catalog.time()
             );
@@ -1599,7 +1631,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
      * @param causalityToken Causality token.
      * @param tableDescriptor Catalog table descriptor.
      * @param zoneDescriptor Catalog distributed zone descriptor.
-     * @param assignmentsFuture Future with assignments.
+     * @param stableAssignmentsFuture Future with assignments.
      * @param onNodeRecovery {@code true} when called during node recovery, {@code false} otherwise.
      * @return Future that will be completed when local changes related to the table creation are applied.
      */
@@ -1607,11 +1639,12 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
             long causalityToken,
             CatalogTableDescriptor tableDescriptor,
             CatalogZoneDescriptor zoneDescriptor,
-            CompletableFuture<List<Assignments>> assignmentsFuture,
+            CompletableFuture<List<Assignments>> stableAssignmentsFuture,
+            List<Assignments> pendingAssignments,
             boolean onNodeRecovery,
             long assignmentsTimestamp
     ) {
-        TableImpl table =  createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
+        TableImpl table = createTableImpl(causalityToken, tableDescriptor, zoneDescriptor);
 
         int tableId = tableDescriptor.id();
 
@@ -1625,7 +1658,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
 
         // NB: all vv.update() calls must be made from the synchronous part of the method (not in thenCompose()/etc!).
         CompletableFuture<?> localPartsUpdateFuture = localPartitionsVv.update(causalityToken,
-                (ignore, throwable) -> inBusyLock(busyLock, () -> assignmentsFuture.thenComposeAsync(newAssignments -> {
+                (ignore, throwable) -> inBusyLock(busyLock, () -> stableAssignmentsFuture.thenComposeAsync(newAssignments -> {
                     PartitionSet parts = new BitSetPartitionSet();
 
                     for (int i = 0; i < newAssignments.size(); i++) {
@@ -1647,6 +1680,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                 return failedFuture(e);
             }
 
+            // `stableAssignmentsFuture` is already completed by this time (it's in chain of `localPartsUpdateFuture`).
             return allOf(localPartsUpdateFuture, tablesByIdFuture).thenComposeAsync(ignore -> inBusyLock(busyLock, () -> {
                         if (onNodeRecovery) {
                             SchemaRegistry schemaRegistry = table.schemaView();
@@ -1657,9 +1691,9 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                             registerIndexesToTable(table, catalogService, partitionSet, schemaRegistry, lwm);
                         }
                         return startLocalPartitionsAndClients(
-                                assignmentsFuture,
+                                stableAssignmentsFuture,
+                                pendingAssignments,
                                 table,
-                                zoneDescriptor.id(),
                                 onNodeRecovery,
                                 assignmentsTimestamp
                         );
@@ -1693,17 +1727,21 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
             assignmentsFuture = completedFuture(
                     tableAssignmentsGetLocally(metaStorageMgr, tableId, zoneDescriptor.partitions(), causalityToken));
         } else {
-
             Catalog catalog = catalogService.catalog(catalogVersion);
 
             long assignmentsTimestamp = catalog.time();
 
             assignmentsFuture = distributionZoneManager.dataNodes(causalityToken, catalogVersion, zoneDescriptor.id())
-                    .thenApply(dataNodes -> PartitionDistributionUtils.calculateAssignments(
-                            dataNodes,
-                            zoneDescriptor.partitions(),
-                            zoneDescriptor.replicas()
-                    ).stream().map(assignments -> Assignments.of(assignments, assignmentsTimestamp)).collect(toList()));
+                    .thenApply(dataNodes ->
+                            PartitionDistributionUtils.calculateAssignments(
+                                            dataNodes,
+                                            zoneDescriptor.partitions(),
+                                            zoneDescriptor.replicas()
+                                    )
+                                    .stream()
+                                    .map(assignments -> Assignments.of(assignments, assignmentsTimestamp))
+                                    .collect(toList())
+                    );
 
             assignmentsFuture.thenAccept(assignmentsList -> LOG.info(
                     "Assignments calculated from data nodes [table={}, tableId={}, assignments={}, revision={}]",
@@ -2097,16 +2135,13 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                             );
                         }
 
-                        long assignmentsTimestamp = pendingAssignments.timestamp();
-
                         return handleChangePendingAssignmentEvent(
                                 replicaGrpId,
                                 table,
                                 stableAssignments,
                                 pendingAssignments,
                                 revision,
-                                isRecovery,
-                                assignmentsTimestamp
+                                isRecovery
                         ).thenAccept(v -> executeIfLocalNodeIsPrimaryForGroup(
                                 replicaGrpId,
                                 replicaMeta -> sendChangePeersAndLearnersRequest(
@@ -2129,17 +2164,26 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
             @Nullable Assignments stableAssignments,
             Assignments pendingAssignments,
             long revision,
-            boolean isRecovery,
-            long assignmentsTimestamp
+            boolean isRecovery
     ) {
         boolean pendingAssignmentsAreForced = pendingAssignments.force();
         Set<Assignment> pendingAssignmentsNodes = pendingAssignments.nodes();
+        long assignmentsTimestamp = pendingAssignments.timestamp();
 
         // Start a new Raft node and Replica if this node has appeared in the new assignments.
-        Assignment localMemberAssignment = localMemberAssignment(pendingAssignments);
-
-        boolean shouldStartLocalGroupNode = localMemberAssignment != null
-                && (stableAssignments == null || !stableAssignments.nodes().contains(localMemberAssignment));
+        Assignment localMemberAssignmentInPending = localMemberAssignment(pendingAssignments);
+        Assignment localMemberAssignmentInStable = localMemberAssignment(stableAssignments);
+
+        boolean shouldStartLocalGroupNode;
+        if (isRecovery) {
+            // The condition to start the replica is
+            // `pending.contains(node) || (stable.contains(node) && !pending.isForce())`.
+            // This condition covers the left part of the OR expression.
+            // The right part of it is covered in `startLocalPartitionsAndClients`.
+            shouldStartLocalGroupNode = localMemberAssignmentInPending != null;
+        } else {
+            shouldStartLocalGroupNode = localMemberAssignmentInPending != null && localMemberAssignmentInStable == null;
+        }
 
         // This is a set of assignments for nodes that are not the part of stable assignments, i.e. unstable part of the distribution.
         // For regular pending assignments we use (old) stable set, so that none of new nodes would be able to propose itself as a leader.
@@ -2152,7 +2196,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
         if (stableAssignments == null || stableAssignments.nodes().isEmpty()) {
             // This condition can only pass if all stable nodes are dead, and we start new raft group from scratch.
             // In this case new initial configuration must match new forced assignments.
-            computedStableAssignments = Assignments.forced(pendingAssignmentsNodes, pendingAssignments.timestamp());
+            computedStableAssignments = Assignments.forced(pendingAssignmentsNodes, assignmentsTimestamp);
         } else if (pendingAssignmentsAreForced) {
             // In case of forced assignments we need to remove nodes that are present in the stable set but are missing from the
             // pending set. Such operation removes dead stable nodes from the resulting stable set, which guarantees that we will
@@ -2188,22 +2232,19 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                         );
 
                         return waitForMetadataCompleteness(assignmentsTimestamp).thenCompose(ignored -> inBusyLock(busyLock, () -> {
-                            int catalogVersion = catalogService.activeCatalogVersion(assignmentsTimestamp);
-
-                            int zoneId = getTableDescriptor(tbl.tableId(), catalogVersion).zoneId();
+                            assert localMemberAssignmentInPending != null : "Local member assignment";
 
                             return startPartitionAndStartClient(
                                     tbl,
                                     replicaGrpId.partitionId(),
-                                    localMemberAssignment,
+                                    localMemberAssignmentInPending,
                                     computedStableAssignments,
-                                    zoneId,
                                     isRecovery,
                                     assignmentsTimestamp
                             );
                         }));
                     }), ioExecutor);
-        } else if (pendingAssignmentsAreForced && localMemberAssignment != null) {
+        } else if (pendingAssignmentsAreForced && localMemberAssignmentInPending != null) {
             localServicesStartFuture = runAsync(() -> inBusyLock(busyLock, () -> {
                 assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group: " + replicaGrpId;
 
@@ -2231,6 +2272,21 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                             : "The local node is outside of the replication group [inStableOrPending=" + isLocalNodeInStableOrPending
                             + ", isLeaseholder=" + isLeaseholder + "].";
 
+                    // A node might exist in stable yet we don't want to start the replica
+                    // The case is: nodes A, B and C hold a replication group,
+                    // nodes A and B die.
+                    // Reset adds C into force pending and user writes data onto C.
+                    // Then A and B go back online. In this case
+                    // stable = [A, B, C], pending = [C, force] and only C should be started.
+                    if (isRecovery && !replicaMgr.isReplicaStarted(replicaGrpId)) {
+                        return;
+                    }
+
+                    assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
+                            + ", stable=" + stableAssignments
+                            + ", pending=" + pendingAssignments
+                            + ", localName=" + localNode().name() + "].";
+
                     // For forced assignments, we exclude dead stable nodes, and all alive stable nodes are already in pending assignments.
                     // Union is not required in such a case.
                     Set<Assignment> newAssignments = pendingAssignmentsAreForced || stableAssignments == null
@@ -2294,7 +2350,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
             Assignments pendingAssignments,
             long revision
     ) {
-        Entry reduceEntry  = metaStorageMgr.getLocally(RebalanceUtil.switchReduceKey(replicaGrpId), revision);
+        Entry reduceEntry = metaStorageMgr.getLocally(RebalanceUtil.switchReduceKey(replicaGrpId), revision);
 
         Assignments reduceAssignments = reduceEntry != null
                 ? Assignments.fromBytes(reduceEntry.value())
@@ -2304,17 +2360,7 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                 ? subtract(stableAssignments.nodes(), reduceAssignments.nodes())
                 : stableAssignments.nodes();
 
-        if (!isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()))) {
-            return false;
-        }
-
-        assert replicaMgr.isReplicaStarted(replicaGrpId) : "The local node is outside of the replication group ["
-                + ", stable=" + stableAssignments
-                + ", pending=" + pendingAssignments
-                + ", reduce=" + reduceAssignments
-                + ", localName=" + localNode().name() + "].";
-
-        return true;
+        return isLocalNodeInAssignments(union(reducedStableAssignments, pendingAssignments.nodes()));
     }
 
     private SnapshotStorageFactory createSnapshotStorageFactory(
@@ -2883,16 +2929,18 @@ public class TableManager implements IgniteTablesInternal, IgniteComponent {
                 assert stableAssignments != null : "tablePartitionId=" + tablePartitionId + ", revision=" + revision;
 
                 return waitForMetadataCompleteness(assignmentsTimestamp).thenCompose(unused2 -> inBusyLockAsync(busyLock, () -> {
-                    int catalogVersion = catalogService.activeCatalogVersion(assignmentsTimestamp);
+                    Assignment localMemberAssignment = localMemberAssignment(stableAssignments);
 
-                    int zoneId = getTableDescriptor(tablePartitionId.tableId(), catalogVersion).zoneId();
+                    if (localMemberAssignment == null) {
+                        // (0) in case if node not in the assignments
+                        return nullCompletedFuture();
+                    }
 
                     return startPartitionAndStartClient(
                             table,
                             tablePartitionId.partitionId(),
-                            localMemberAssignment(stableAssignments),
+                            localMemberAssignment,
                             stableAssignments,
-                            zoneId,
                             false,
                             assignmentsTimestamp
                     );