You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sd...@apache.org on 2022/08/16 16:54:19 UTC

[ignite-3] 01/02: .

This is an automated email from the ASF dual-hosted git repository.

sdanilov pushed a commit to branch ignite-17196
in repository https://gitbox.apache.org/repos/asf/ignite-3.git

commit c17b1f654001085e7de5305b502ccd8c15bf5a30
Author: Semyon Danilov <sa...@yandex.ru>
AuthorDate: Mon Aug 1 15:45:06 2022 +0400

    .
---
 .../runner/app/ItIgniteNodeRestartTest.java        |   2 +
 .../internal/table/distributed/TableManager.java   |  31 ++-
 .../raft/RebalanceRaftGroupEventsListener.java     |  37 +--
 .../ignite/internal/utils/RebalanceUtil.java       | 247 +++++++++++++++++++++
 4 files changed, 289 insertions(+), 28 deletions(-)

diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
index 89c3e30f59..5f9d64156f 100644
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
@@ -74,6 +74,7 @@ import org.apache.ignite.internal.schema.SchemaManager;
 import org.apache.ignite.internal.storage.DataStorageManager;
 import org.apache.ignite.internal.storage.DataStorageModule;
 import org.apache.ignite.internal.storage.DataStorageModules;
+import org.apache.ignite.internal.storage.pagememory.configuration.schema.VolatilePageMemoryDataStorageChange;
 import org.apache.ignite.internal.table.TableImpl;
 import org.apache.ignite.internal.table.distributed.TableManager;
 import org.apache.ignite.internal.table.distributed.TableTxManagerImpl;
@@ -1031,6 +1032,7 @@ public class ItIgniteNodeRestartTest extends IgniteAbstractTest {
         Table table = ignite.tables().createTable(
                 scmTbl1.canonicalName(),
                 tbl -> convert(scmTbl1, tbl).changeReplicas(replicas).changePartitions(partitions)
+                        .changeDataStorage(dsc -> dsc.convert(VolatilePageMemoryDataStorageChange.class))
         );
 
         for (int i = 0; i < 100; i++) {
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index 1d8cb9b2a8..6c433a19d4 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -561,7 +561,13 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
             });
         }
 
-        CompletableFuture.allOf(futures).join();
+    private void onRebalanceDone(List<PeerId> newPeers, int partNum, String partId) {
+        UUID tblId = extractTableId(new ByteArray(partId), ASSIGNMENTS_SWITCH_REDUCE_PREFIX);
+        TableImpl tbl = tablesByIdVv.latest().get(tblId);
+        ExtendedTableConfiguration tblCfg = (ExtendedTableConfiguration) tablesCfg.tables().get(tbl.name());
+        Integer partitions = tblCfg.partitions().value();
+        Integer replicas = tblCfg.replicas().value();
+        RebalanceUtil.onRebalanceDone(baselineMgr.nodes(), partitions, replicas, newPeers, metaStorageMgr, partNum, partId).join();
     }
 
     private RaftGroupOptions groupOptionsForPartition(
@@ -1481,6 +1487,29 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
                 LOG.warn("Unable to process stable assignments event", e);
             }
         });
+
+        metaStorageMgr.registerWatchByPrefix(ByteArray.fromString(ASSIGNMENTS_SWITCH_REDUCE_PREFIX), new WatchListener() {
+            @Override
+            public boolean onUpdate(@NotNull WatchEvent evt) {
+                ByteArray key = evt.entryEvent().newEntry().key();
+                int partitionNumber = extractPartitionNumber(key);
+                UUID tblId = extractTableId(key, ASSIGNMENTS_SWITCH_REDUCE_PREFIX);
+                String partitionId = partitionRaftGroupName(tblId, partitionNumber);
+                TableImpl tbl = tablesByIdVv.latest().get(tblId);
+                ExtendedTableConfiguration tblCfg = (ExtendedTableConfiguration) tablesCfg.tables().get(tbl.name());
+                Integer partitions = tblCfg.partitions().value();
+                Integer replicas = tblCfg.replicas().value();
+                RebalanceUtil.handleReduceChanged(metaStorageMgr, baselineMgr.nodes(), partitions, replicas, partitionNumber,
+                        partitionId, evt).join();
+
+                return true;
+            }
+
+            @Override
+            public void onError(@NotNull Throwable e) {
+                LOG.warn("Unable to process stable assignments event", e);
+            }
+        });
     }
 
     /**
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java
index bd562157e5..89304ae396 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java
@@ -24,10 +24,10 @@ import static org.apache.ignite.internal.metastorage.client.Operations.put;
 import static org.apache.ignite.internal.metastorage.client.Operations.remove;
 import static org.apache.ignite.internal.utils.RebalanceUtil.pendingPartAssignmentsKey;
 import static org.apache.ignite.internal.utils.RebalanceUtil.plannedPartAssignmentsKey;
+import static org.apache.ignite.internal.utils.RebalanceUtil.resolveClusterNodes;
 import static org.apache.ignite.internal.utils.RebalanceUtil.stablePartAssignmentsKey;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -48,7 +48,7 @@ import org.apache.ignite.internal.raft.server.RaftGroupEventsListener;
 import org.apache.ignite.internal.util.ByteUtils;
 import org.apache.ignite.internal.util.IgniteSpinBusyLock;
 import org.apache.ignite.lang.ByteArray;
-import org.apache.ignite.lang.IgniteInternalException;
+import org.apache.ignite.lang.IgniteTriConsumer;
 import org.apache.ignite.network.ClusterNode;
 import org.apache.ignite.network.NetworkAddress;
 import org.apache.ignite.raft.client.Peer;
@@ -94,6 +94,8 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
     /** Delay between unsuccessful trial of a rebalance and a new trial, ms. */
     public static final int REBALANCE_RETRY_DELAY_MS = 200;
 
+    private IgniteTriConsumer<List<PeerId>, Integer, String> rebalanceDoneListener;
+
     /**
      * Constructs new listener.
      *
@@ -110,6 +112,7 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
             int partNum,
             IgniteSpinBusyLock busyLock,
             BiFunction<List<Peer>, Long, CompletableFuture<Void>> movePartitionFn,
+            IgniteTriConsumer<List<PeerId>, Integer, String> rebalanceDoneListener,
             ScheduledExecutorService rebalanceScheduler) {
         this.metaStorageMgr = metaStorageMgr;
         this.tblConfiguration = tblConfiguration;
@@ -117,6 +120,7 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
         this.partNum = partNum;
         this.busyLock = busyLock;
         this.movePartitionFn = movePartitionFn;
+        this.rebalanceDoneListener = rebalanceDoneListener;
         this.rebalanceScheduler = rebalanceScheduler;
     }
 
@@ -299,6 +303,8 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
                         partNum, tblConfiguration.name().value(), appliedPeers);
             }
 
+            this.onRebalanceDone(peers);
+
             rebalanceAttempts.set(0);
         } catch (InterruptedException | ExecutionException e) {
             // TODO: IGNITE-14693
@@ -307,31 +313,8 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
         }
     }
 
-    private static List<ClusterNode> resolveClusterNodes(
-            List<PeerId> peers, byte[] pendingAssignments, byte[] stableAssignments) {
-        Map<NetworkAddress, ClusterNode> resolveRegistry = new HashMap<>();
-
-        if (pendingAssignments != null) {
-            ((List<ClusterNode>) ByteUtils.fromBytes(pendingAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
-        }
-
-        if (stableAssignments != null) {
-            ((List<ClusterNode>) ByteUtils.fromBytes(stableAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
-        }
-
-        List<ClusterNode> resolvedNodes = new ArrayList<>(peers.size());
-
-        for (PeerId p : peers) {
-            var addr = NetworkAddress.from(p.getEndpoint().getIp() + ":" + p.getEndpoint().getPort());
-
-            if (resolveRegistry.containsKey(addr)) {
-                resolvedNodes.add(resolveRegistry.get(addr));
-            } else {
-                throw new IgniteInternalException("Can't find appropriate cluster node for raft group peer: " + p);
-            }
-        }
-
-        return resolvedNodes;
+    private void onRebalanceDone(List<PeerId> newPeers) {
+        this.rebalanceDoneListener.accept(newPeers, partNum, partId);
     }
 
     /**
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java b/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java
index 9f0f7db412..aaf85663c4 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java
@@ -20,22 +20,38 @@ package org.apache.ignite.internal.utils;
 import static org.apache.ignite.internal.metastorage.client.CompoundCondition.and;
 import static org.apache.ignite.internal.metastorage.client.CompoundCondition.or;
 import static org.apache.ignite.internal.metastorage.client.Conditions.notExists;
+import static org.apache.ignite.internal.metastorage.client.Conditions.revision;
 import static org.apache.ignite.internal.metastorage.client.Conditions.value;
 import static org.apache.ignite.internal.metastorage.client.Operations.ops;
 import static org.apache.ignite.internal.metastorage.client.Operations.put;
 import static org.apache.ignite.internal.metastorage.client.Operations.remove;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 import org.apache.ignite.internal.affinity.AffinityUtils;
 import org.apache.ignite.internal.logger.IgniteLogger;
 import org.apache.ignite.internal.logger.Loggers;
 import org.apache.ignite.internal.metastorage.MetaStorageManager;
+import org.apache.ignite.internal.metastorage.client.Condition;
+import org.apache.ignite.internal.metastorage.client.Conditions;
+import org.apache.ignite.internal.metastorage.client.Entry;
 import org.apache.ignite.internal.metastorage.client.If;
+import org.apache.ignite.internal.metastorage.client.Operations;
+import org.apache.ignite.internal.metastorage.client.WatchEvent;
 import org.apache.ignite.internal.util.ByteUtils;
 import org.apache.ignite.lang.ByteArray;
+import org.apache.ignite.lang.IgniteInternalException;
 import org.apache.ignite.network.ClusterNode;
+import org.apache.ignite.network.NetworkAddress;
+import org.apache.ignite.raft.jraft.entity.PeerId;
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -235,4 +251,235 @@ public class RebalanceUtil {
         // As long as we don't have a general failure handler, we assume that all errors are recoverable.
         return true;
     }
+
+    public static final String ASSIGNMENTS_SWITCH_REDUCE_PREFIX = "assignments.switch.reduce.";
+
+    private static ByteArray switchReduce(String partId) {
+        return new ByteArray(ASSIGNMENTS_SWITCH_REDUCE_PREFIX + partId);
+    }
+
+    private static ByteArray switchAppend(String partId) {
+        return new ByteArray("assignments.switch.append." + partId);
+    }
+
+    public static CompletableFuture<Void> stepDownAsPeer(String partId, ClusterNode clusterNode, MetaStorageManager metaStorageMgr) {
+        ByteArray key = switchReduce(partId);
+
+        return metaStorageMgr.get(key)
+                .thenCompose(retrievedAssignmentsSwitchReduce -> {
+                    byte[] prevValue = retrievedAssignmentsSwitchReduce.value();
+
+                    boolean prevValueEmpty = true;
+                    List<ClusterNode> calculatedAssignmentsSwitchReduce;
+
+                    if (prevValue != null) {
+                        calculatedAssignmentsSwitchReduce = (List<ClusterNode>) ByteUtils.fromBytes(prevValue);
+                        prevValueEmpty = false;
+                    } else {
+                        calculatedAssignmentsSwitchReduce = new ArrayList<>();
+                    }
+
+                    calculatedAssignmentsSwitchReduce.add(clusterNode);
+
+                    byte[] newValue = ByteUtils.toBytes(calculatedAssignmentsSwitchReduce);
+
+                    if (prevValueEmpty) {
+                        return metaStorageMgr.invoke(
+                                Conditions.notExists(key),
+                                Operations.put(key, newValue),
+                                Operations.noop()
+                        );
+                    } else {
+                        return metaStorageMgr.invoke(
+                                revision(key).eq(retrievedAssignmentsSwitchReduce.revision()),
+                                Operations.put(key, newValue),
+                                Operations.noop()
+                        );
+                    }
+                }).thenCompose(val -> {
+                    if (!val) {
+                        return stepDownAsPeer(partId, clusterNode, metaStorageMgr);
+                    }
+
+                    return CompletableFuture.completedFuture(null);
+                });
+    }
+
+    private static List<ClusterNode> subtract(Collection<ClusterNode> minuend, Collection<ClusterNode> subtrahend) {
+        return minuend.stream().filter(v -> !subtrahend.contains(v)).collect(Collectors.toList());
+    }
+
+    private static List<ClusterNode> union(Collection<ClusterNode> op1, Collection<ClusterNode> op2) {
+        return op2.stream().filter(v -> !op1.contains(v)).collect(Collectors.toCollection(() -> {
+            return new ArrayList<>(op1);
+        }));
+    }
+
+    private static List<ClusterNode> intersect(Collection<ClusterNode> op1, Collection<ClusterNode> op2) {
+        return op1.stream().filter(op2::contains).collect(Collectors.toList());
+    }
+
+    public static CompletableFuture<Void> handleReduceChanged(MetaStorageManager metaStorageMgr, Collection<ClusterNode> baselineNodes,
+            int partitions, int replicas, int partNum, String partId, WatchEvent event) {
+        Entry entry = event.entryEvent().newEntry();
+        byte[] eventData = entry.value();
+
+        List<ClusterNode> ass = AffinityUtils.calculateAssignments(baselineNodes, partitions, replicas).get(partNum);
+        List<ClusterNode> assSwitchReduce = (List<ClusterNode>) ByteUtils.fromBytes(eventData);
+
+        ByteArray partAssignmentsPendingKey = pendingPartAssignmentsKey(partId);
+
+        ass = subtract(ass, assSwitchReduce);
+
+        byte[] calculatedAssignments = ByteUtils.toBytes(ass);
+
+        ByteArray partChangeTriggerKey = partChangeTriggerKey(partId);
+        byte[] rev = ByteUtils.longToBytes(event.entryEvent().newEntry().revision());
+
+        If iif = If.iif(
+                and(
+                        or(notExists(partChangeTriggerKey), value(partChangeTriggerKey).lt(rev)),
+                        notExists(partAssignmentsPendingKey)
+                ),
+                Operations.ops(
+                        put(partAssignmentsPendingKey, calculatedAssignments),
+                        put(partChangeTriggerKey, rev)
+                ).yield(),
+                Operations.ops().yield()
+        );
+
+        return metaStorageMgr.invoke(iif).thenApply(unused -> null);
+    }
+
+    public static CompletableFuture<Void> onRebalanceDone(Collection<ClusterNode> baselineNodes, int partitions, int replicas,
+            List<PeerId> newPeers, MetaStorageManager metaStorageMgr, int partNum, String partId) {
+
+        ByteArray pendingPartAssignmentsKey = pendingPartAssignmentsKey(partId);
+        ByteArray stablePartAssignmentsKey = stablePartAssignmentsKey(partId);
+        ByteArray plannedPartAssignmentsKey = plannedPartAssignmentsKey(partId);
+        ByteArray switchReduceKey = switchReduce(partId);
+        ByteArray switchAppendKey = switchAppend(partId);
+
+        CompletableFuture<Map<ByteArray, Entry>> all = metaStorageMgr.getAll(Set.of(
+                pendingPartAssignmentsKey,
+                stablePartAssignmentsKey,
+                plannedPartAssignmentsKey,
+                switchReduceKey,
+                switchAppendKey
+        ));
+
+        return all.thenCompose(resMap -> {
+            var retrievedAssignmentsStable = resMap.get(stablePartAssignmentsKey);
+            var retrievedAssignmentsPending = resMap.get(pendingPartAssignmentsKey);
+            var retrievedAssignmentsPlanned = resMap.get(plannedPartAssignmentsKey);
+            var retrievedAssignmentsSwitchReduce = resMap.get(switchReduceKey);
+            var retrievedAssignmentsSwitchAppend = resMap.get(switchAppendKey);
+
+            List<ClusterNode> resolvedStable = resolveClusterNodes(newPeers, retrievedAssignmentsPending.value(),
+                    retrievedAssignmentsStable.value());
+
+            List<ClusterNode> retrievedAssignmentsSwitchReduce0 = (List<ClusterNode>) ByteUtils.fromBytes(retrievedAssignmentsSwitchReduce.value());
+            List<ClusterNode> retrievedAssignmentsSwitchAppend0 = (List<ClusterNode>) ByteUtils.fromBytes(retrievedAssignmentsSwitchAppend.value());
+            List<ClusterNode> retrievedAssignmentsStable0 = (List<ClusterNode>) ByteUtils.fromBytes(retrievedAssignmentsStable.value());
+
+            List<ClusterNode> calculatedAssignments = AffinityUtils.calculateAssignments(baselineNodes, partitions, replicas).get(partNum);
+
+            // Were reduced
+            List<ClusterNode> reducedNodes = subtract(retrievedAssignmentsSwitchReduce0, resolvedStable);
+
+            // Were added
+            List<ClusterNode> addedNodes = subtract(resolvedStable, retrievedAssignmentsStable0);
+
+            // For further reduction
+            List<ClusterNode> calculatedAssignmentsSwitchReduce = subtract(retrievedAssignmentsSwitchReduce0, reducedNodes);
+
+            // For further addition
+            List<ClusterNode> calculatedAssignmentsSwitchAppend = union(retrievedAssignmentsSwitchAppend0, reducedNodes);
+            calculatedAssignmentsSwitchAppend = subtract(calculatedAssignmentsSwitchAppend, addedNodes);
+            calculatedAssignmentsSwitchAppend = intersect(calculatedAssignments, calculatedAssignmentsSwitchAppend);
+
+            var calculatedPendingReduction = subtract(resolvedStable, retrievedAssignmentsSwitchReduce0);
+
+            var calculatedPendingAddition = union(resolvedStable, reducedNodes);
+            calculatedPendingAddition = intersect(calculatedAssignments, calculatedPendingAddition);
+
+            // retry preconditions should be checked inside every invoke.
+
+            Condition retryPreconditions = and(
+                    revision(stablePartAssignmentsKey).eq(retrievedAssignmentsStable.revision()),
+                    and(
+                            revision(pendingPartAssignmentsKey).eq(retrievedAssignmentsPending.revision()),
+                            and(
+                                    revision(switchReduceKey).eq(retrievedAssignmentsSwitchReduce.revision()),
+                                    revision(switchAppendKey).eq(retrievedAssignmentsSwitchAppend.revision())
+                            )
+                    )
+            );
+
+            Operations ops;
+
+            var rs = ByteUtils.toBytes(resolvedStable);
+            var cpa = ByteUtils.toBytes(calculatedPendingAddition);
+            var cpr = ByteUtils.toBytes(calculatedPendingReduction);
+            var casr = ByteUtils.toBytes(calculatedAssignmentsSwitchReduce);
+            var casa = ByteUtils.toBytes(calculatedAssignmentsSwitchAppend);
+
+            if (!calculatedAssignmentsSwitchAppend.isEmpty()) {
+                ops = ops(
+                        put(stablePartAssignmentsKey, rs),
+                        put(pendingPartAssignmentsKey, cpa),
+                        put(switchReduceKey, casr),
+                        put(switchAppendKey, casa)
+                );
+            } else if (!calculatedAssignmentsSwitchReduce.isEmpty()) {
+                ops = ops(
+                        put(stablePartAssignmentsKey, rs),
+                        put(pendingPartAssignmentsKey, cpr),
+                        put(switchReduceKey, casr),
+                        put(switchAppendKey, casa)
+                );
+            } else {
+                ops = ops(
+                        put(pendingPartAssignmentsKey, retrievedAssignmentsPlanned.value()),
+                        remove(plannedPartAssignmentsKey),
+                        put(switchAppendKey, casa)
+                );
+            }
+
+            return metaStorageMgr.invoke(If.iif(retryPreconditions, ops.yield(true), ops().yield(false))).thenCompose(statementResult -> {
+                if (statementResult.getAsBoolean()) {
+                    return CompletableFuture.completedFuture(null);
+                } else {
+                    return onRebalanceDone(baselineNodes, partitions, replicas, newPeers, metaStorageMgr, partNum, partId);
+                }
+            });
+        });
+    }
+
+    public static List<ClusterNode> resolveClusterNodes(
+            List<PeerId> peers, byte[] pendingAssignments, byte[] stableAssignments) {
+        Map<NetworkAddress, ClusterNode> resolveRegistry = new HashMap<>();
+
+        if (pendingAssignments != null) {
+            ((List<ClusterNode>) ByteUtils.fromBytes(pendingAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
+        }
+
+        if (stableAssignments != null) {
+            ((List<ClusterNode>) ByteUtils.fromBytes(stableAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
+        }
+
+        List<ClusterNode> resolvedNodes = new ArrayList<>(peers.size());
+
+        for (PeerId p : peers) {
+            var addr = NetworkAddress.from(p.getEndpoint().getIp() + ":" + p.getEndpoint().getPort());
+
+            if (resolveRegistry.containsKey(addr)) {
+                resolvedNodes.add(resolveRegistry.get(addr));
+            } else {
+                throw new IgniteInternalException("Can't find appropriate cluster node for raft group peer: " + p);
+            }
+        }
+
+        return resolvedNodes;
+    }
 }