You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sd...@apache.org on 2022/08/16 16:54:19 UTC
[ignite-3] 01/02: .
This is an automated email from the ASF dual-hosted git repository.
sdanilov pushed a commit to branch ignite-17196
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit c17b1f654001085e7de5305b502ccd8c15bf5a30
Author: Semyon Danilov <sa...@yandex.ru>
AuthorDate: Mon Aug 1 15:45:06 2022 +0400
.
---
.../runner/app/ItIgniteNodeRestartTest.java | 2 +
.../internal/table/distributed/TableManager.java | 31 ++-
.../raft/RebalanceRaftGroupEventsListener.java | 37 +--
.../ignite/internal/utils/RebalanceUtil.java | 247 +++++++++++++++++++++
4 files changed, 289 insertions(+), 28 deletions(-)
diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
index 89c3e30f59..5f9d64156f 100644
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
@@ -74,6 +74,7 @@ import org.apache.ignite.internal.schema.SchemaManager;
import org.apache.ignite.internal.storage.DataStorageManager;
import org.apache.ignite.internal.storage.DataStorageModule;
import org.apache.ignite.internal.storage.DataStorageModules;
+import org.apache.ignite.internal.storage.pagememory.configuration.schema.VolatilePageMemoryDataStorageChange;
import org.apache.ignite.internal.table.TableImpl;
import org.apache.ignite.internal.table.distributed.TableManager;
import org.apache.ignite.internal.table.distributed.TableTxManagerImpl;
@@ -1031,6 +1032,7 @@ public class ItIgniteNodeRestartTest extends IgniteAbstractTest {
Table table = ignite.tables().createTable(
scmTbl1.canonicalName(),
tbl -> convert(scmTbl1, tbl).changeReplicas(replicas).changePartitions(partitions)
+ .changeDataStorage(dsc -> dsc.convert(VolatilePageMemoryDataStorageChange.class))
);
for (int i = 0; i < 100; i++) {
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
index 1d8cb9b2a8..6c433a19d4 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java
@@ -561,7 +561,13 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
});
}
- CompletableFuture.allOf(futures).join();
+ private void onRebalanceDone(List<PeerId> newPeers, int partNum, String partId) {
+ UUID tblId = extractTableId(new ByteArray(partId), ASSIGNMENTS_SWITCH_REDUCE_PREFIX);
+ TableImpl tbl = tablesByIdVv.latest().get(tblId);
+ ExtendedTableConfiguration tblCfg = (ExtendedTableConfiguration) tablesCfg.tables().get(tbl.name());
+ Integer partitions = tblCfg.partitions().value();
+ Integer replicas = tblCfg.replicas().value();
+ RebalanceUtil.onRebalanceDone(baselineMgr.nodes(), partitions, replicas, newPeers, metaStorageMgr, partNum, partId).join();
}
private RaftGroupOptions groupOptionsForPartition(
@@ -1481,6 +1487,29 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp
LOG.warn("Unable to process stable assignments event", e);
}
});
+
+ metaStorageMgr.registerWatchByPrefix(ByteArray.fromString(ASSIGNMENTS_SWITCH_REDUCE_PREFIX), new WatchListener() {
+ @Override
+ public boolean onUpdate(@NotNull WatchEvent evt) {
+ ByteArray key = evt.entryEvent().newEntry().key();
+ int partitionNumber = extractPartitionNumber(key);
+ UUID tblId = extractTableId(key, ASSIGNMENTS_SWITCH_REDUCE_PREFIX);
+ String partitionId = partitionRaftGroupName(tblId, partitionNumber);
+ TableImpl tbl = tablesByIdVv.latest().get(tblId);
+ ExtendedTableConfiguration tblCfg = (ExtendedTableConfiguration) tablesCfg.tables().get(tbl.name());
+ Integer partitions = tblCfg.partitions().value();
+ Integer replicas = tblCfg.replicas().value();
+ RebalanceUtil.handleReduceChanged(metaStorageMgr, baselineMgr.nodes(), partitions, replicas, partitionNumber,
+ partitionId, evt).join();
+
+ return true;
+ }
+
+ @Override
+ public void onError(@NotNull Throwable e) {
+ LOG.warn("Unable to process stable assignments event", e);
+ }
+ });
}
/**
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java
index bd562157e5..89304ae396 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/raft/RebalanceRaftGroupEventsListener.java
@@ -24,10 +24,10 @@ import static org.apache.ignite.internal.metastorage.client.Operations.put;
import static org.apache.ignite.internal.metastorage.client.Operations.remove;
import static org.apache.ignite.internal.utils.RebalanceUtil.pendingPartAssignmentsKey;
import static org.apache.ignite.internal.utils.RebalanceUtil.plannedPartAssignmentsKey;
+import static org.apache.ignite.internal.utils.RebalanceUtil.resolveClusterNodes;
import static org.apache.ignite.internal.utils.RebalanceUtil.stablePartAssignmentsKey;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -48,7 +48,7 @@ import org.apache.ignite.internal.raft.server.RaftGroupEventsListener;
import org.apache.ignite.internal.util.ByteUtils;
import org.apache.ignite.internal.util.IgniteSpinBusyLock;
import org.apache.ignite.lang.ByteArray;
-import org.apache.ignite.lang.IgniteInternalException;
+import org.apache.ignite.lang.IgniteTriConsumer;
import org.apache.ignite.network.ClusterNode;
import org.apache.ignite.network.NetworkAddress;
import org.apache.ignite.raft.client.Peer;
@@ -94,6 +94,8 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
/** Delay between unsuccessful trial of a rebalance and a new trial, ms. */
public static final int REBALANCE_RETRY_DELAY_MS = 200;
+ private IgniteTriConsumer<List<PeerId>, Integer, String> rebalanceDoneListener;
+
/**
* Constructs new listener.
*
@@ -110,6 +112,7 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
int partNum,
IgniteSpinBusyLock busyLock,
BiFunction<List<Peer>, Long, CompletableFuture<Void>> movePartitionFn,
+ IgniteTriConsumer<List<PeerId>, Integer, String> rebalanceDoneListener,
ScheduledExecutorService rebalanceScheduler) {
this.metaStorageMgr = metaStorageMgr;
this.tblConfiguration = tblConfiguration;
@@ -117,6 +120,7 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
this.partNum = partNum;
this.busyLock = busyLock;
this.movePartitionFn = movePartitionFn;
+ this.rebalanceDoneListener = rebalanceDoneListener;
this.rebalanceScheduler = rebalanceScheduler;
}
@@ -299,6 +303,8 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
partNum, tblConfiguration.name().value(), appliedPeers);
}
+ this.onRebalanceDone(peers);
+
rebalanceAttempts.set(0);
} catch (InterruptedException | ExecutionException e) {
// TODO: IGNITE-14693
@@ -307,31 +313,8 @@ public class RebalanceRaftGroupEventsListener implements RaftGroupEventsListener
}
}
- private static List<ClusterNode> resolveClusterNodes(
- List<PeerId> peers, byte[] pendingAssignments, byte[] stableAssignments) {
- Map<NetworkAddress, ClusterNode> resolveRegistry = new HashMap<>();
-
- if (pendingAssignments != null) {
- ((List<ClusterNode>) ByteUtils.fromBytes(pendingAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
- }
-
- if (stableAssignments != null) {
- ((List<ClusterNode>) ByteUtils.fromBytes(stableAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
- }
-
- List<ClusterNode> resolvedNodes = new ArrayList<>(peers.size());
-
- for (PeerId p : peers) {
- var addr = NetworkAddress.from(p.getEndpoint().getIp() + ":" + p.getEndpoint().getPort());
-
- if (resolveRegistry.containsKey(addr)) {
- resolvedNodes.add(resolveRegistry.get(addr));
- } else {
- throw new IgniteInternalException("Can't find appropriate cluster node for raft group peer: " + p);
- }
- }
-
- return resolvedNodes;
+ private void onRebalanceDone(List<PeerId> newPeers) {
+ this.rebalanceDoneListener.accept(newPeers, partNum, partId);
}
/**
diff --git a/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java b/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java
index 9f0f7db412..aaf85663c4 100644
--- a/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java
+++ b/modules/table/src/main/java/org/apache/ignite/internal/utils/RebalanceUtil.java
@@ -20,22 +20,38 @@ package org.apache.ignite.internal.utils;
import static org.apache.ignite.internal.metastorage.client.CompoundCondition.and;
import static org.apache.ignite.internal.metastorage.client.CompoundCondition.or;
import static org.apache.ignite.internal.metastorage.client.Conditions.notExists;
+import static org.apache.ignite.internal.metastorage.client.Conditions.revision;
import static org.apache.ignite.internal.metastorage.client.Conditions.value;
import static org.apache.ignite.internal.metastorage.client.Operations.ops;
import static org.apache.ignite.internal.metastorage.client.Operations.put;
import static org.apache.ignite.internal.metastorage.client.Operations.remove;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
import org.apache.ignite.internal.affinity.AffinityUtils;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
+import org.apache.ignite.internal.metastorage.client.Condition;
+import org.apache.ignite.internal.metastorage.client.Conditions;
+import org.apache.ignite.internal.metastorage.client.Entry;
import org.apache.ignite.internal.metastorage.client.If;
+import org.apache.ignite.internal.metastorage.client.Operations;
+import org.apache.ignite.internal.metastorage.client.WatchEvent;
import org.apache.ignite.internal.util.ByteUtils;
import org.apache.ignite.lang.ByteArray;
+import org.apache.ignite.lang.IgniteInternalException;
import org.apache.ignite.network.ClusterNode;
+import org.apache.ignite.network.NetworkAddress;
+import org.apache.ignite.raft.jraft.entity.PeerId;
import org.jetbrains.annotations.NotNull;
/**
@@ -235,4 +251,235 @@ public class RebalanceUtil {
// As long as we don't have a general failure handler, we assume that all errors are recoverable.
return true;
}
+
+ public static final String ASSIGNMENTS_SWITCH_REDUCE_PREFIX = "assignments.switch.reduce.";
+
+ private static ByteArray switchReduce(String partId) {
+ return new ByteArray(ASSIGNMENTS_SWITCH_REDUCE_PREFIX + partId);
+ }
+
+ private static ByteArray switchAppend(String partId) {
+ return new ByteArray("assignments.switch.append." + partId);
+ }
+
+ public static CompletableFuture<Void> stepDownAsPeer(String partId, ClusterNode clusterNode, MetaStorageManager metaStorageMgr) {
+ ByteArray key = switchReduce(partId);
+
+ return metaStorageMgr.get(key)
+ .thenCompose(retrievedAssignmentsSwitchReduce -> {
+ byte[] prevValue = retrievedAssignmentsSwitchReduce.value();
+
+ boolean prevValueEmpty = true;
+ List<ClusterNode> calculatedAssignmentsSwitchReduce;
+
+ if (prevValue != null) {
+ calculatedAssignmentsSwitchReduce = (List<ClusterNode>) ByteUtils.fromBytes(prevValue);
+ prevValueEmpty = false;
+ } else {
+ calculatedAssignmentsSwitchReduce = new ArrayList<>();
+ }
+
+ calculatedAssignmentsSwitchReduce.add(clusterNode);
+
+ byte[] newValue = ByteUtils.toBytes(calculatedAssignmentsSwitchReduce);
+
+ if (prevValueEmpty) {
+ return metaStorageMgr.invoke(
+ Conditions.notExists(key),
+ Operations.put(key, newValue),
+ Operations.noop()
+ );
+ } else {
+ return metaStorageMgr.invoke(
+ revision(key).eq(retrievedAssignmentsSwitchReduce.revision()),
+ Operations.put(key, newValue),
+ Operations.noop()
+ );
+ }
+ }).thenCompose(val -> {
+ if (!val) {
+ return stepDownAsPeer(partId, clusterNode, metaStorageMgr);
+ }
+
+ return CompletableFuture.completedFuture(null);
+ });
+ }
+
+ private static List<ClusterNode> subtract(Collection<ClusterNode> minuend, Collection<ClusterNode> subtrahend) {
+ return minuend.stream().filter(v -> !subtrahend.contains(v)).collect(Collectors.toList());
+ }
+
+ private static List<ClusterNode> union(Collection<ClusterNode> op1, Collection<ClusterNode> op2) {
+ return op2.stream().filter(v -> !op1.contains(v)).collect(Collectors.toCollection(() -> {
+ return new ArrayList<>(op1);
+ }));
+ }
+
+ private static List<ClusterNode> intersect(Collection<ClusterNode> op1, Collection<ClusterNode> op2) {
+ return op1.stream().filter(op2::contains).collect(Collectors.toList());
+ }
+
+ public static CompletableFuture<Void> handleReduceChanged(MetaStorageManager metaStorageMgr, Collection<ClusterNode> baselineNodes,
+ int partitions, int replicas, int partNum, String partId, WatchEvent event) {
+ Entry entry = event.entryEvent().newEntry();
+ byte[] eventData = entry.value();
+
+ List<ClusterNode> ass = AffinityUtils.calculateAssignments(baselineNodes, partitions, replicas).get(partNum);
+ List<ClusterNode> assSwitchReduce = (List<ClusterNode>) ByteUtils.fromBytes(eventData);
+
+ ByteArray partAssignmentsPendingKey = pendingPartAssignmentsKey(partId);
+
+ ass = subtract(ass, assSwitchReduce);
+
+ byte[] calculatedAssignments = ByteUtils.toBytes(ass);
+
+ ByteArray partChangeTriggerKey = partChangeTriggerKey(partId);
+ byte[] rev = ByteUtils.longToBytes(event.entryEvent().newEntry().revision());
+
+ If iif = If.iif(
+ and(
+ or(notExists(partChangeTriggerKey), value(partChangeTriggerKey).lt(rev)),
+ notExists(partAssignmentsPendingKey)
+ ),
+ Operations.ops(
+ put(partAssignmentsPendingKey, calculatedAssignments),
+ put(partChangeTriggerKey, rev)
+ ).yield(),
+ Operations.ops().yield()
+ );
+
+ return metaStorageMgr.invoke(iif).thenApply(unused -> null);
+ }
+
+ public static CompletableFuture<Void> onRebalanceDone(Collection<ClusterNode> baselineNodes, int partitions, int replicas,
+ List<PeerId> newPeers, MetaStorageManager metaStorageMgr, int partNum, String partId) {
+
+ ByteArray pendingPartAssignmentsKey = pendingPartAssignmentsKey(partId);
+ ByteArray stablePartAssignmentsKey = stablePartAssignmentsKey(partId);
+ ByteArray plannedPartAssignmentsKey = plannedPartAssignmentsKey(partId);
+ ByteArray switchReduceKey = switchReduce(partId);
+ ByteArray switchAppendKey = switchAppend(partId);
+
+ CompletableFuture<Map<ByteArray, Entry>> all = metaStorageMgr.getAll(Set.of(
+ pendingPartAssignmentsKey,
+ stablePartAssignmentsKey,
+ plannedPartAssignmentsKey,
+ switchReduceKey,
+ switchAppendKey
+ ));
+
+ return all.thenCompose(resMap -> {
+ var retrievedAssignmentsStable = resMap.get(stablePartAssignmentsKey);
+ var retrievedAssignmentsPending = resMap.get(pendingPartAssignmentsKey);
+ var retrievedAssignmentsPlanned = resMap.get(plannedPartAssignmentsKey);
+ var retrievedAssignmentsSwitchReduce = resMap.get(switchReduceKey);
+ var retrievedAssignmentsSwitchAppend = resMap.get(switchAppendKey);
+
+ List<ClusterNode> resolvedStable = resolveClusterNodes(newPeers, retrievedAssignmentsPending.value(),
+ retrievedAssignmentsStable.value());
+
+ List<ClusterNode> retrievedAssignmentsSwitchReduce0 = (List<ClusterNode>) ByteUtils.fromBytes(retrievedAssignmentsSwitchReduce.value());
+ List<ClusterNode> retrievedAssignmentsSwitchAppend0 = (List<ClusterNode>) ByteUtils.fromBytes(retrievedAssignmentsSwitchAppend.value());
+ List<ClusterNode> retrievedAssignmentsStable0 = (List<ClusterNode>) ByteUtils.fromBytes(retrievedAssignmentsStable.value());
+
+ List<ClusterNode> calculatedAssignments = AffinityUtils.calculateAssignments(baselineNodes, partitions, replicas).get(partNum);
+
+ // Were reduced
+ List<ClusterNode> reducedNodes = subtract(retrievedAssignmentsSwitchReduce0, resolvedStable);
+
+ // Were added
+ List<ClusterNode> addedNodes = subtract(resolvedStable, retrievedAssignmentsStable0);
+
+ // For further reduction
+ List<ClusterNode> calculatedAssignmentsSwitchReduce = subtract(retrievedAssignmentsSwitchReduce0, reducedNodes);
+
+ // For further addition
+ List<ClusterNode> calculatedAssignmentsSwitchAppend = union(retrievedAssignmentsSwitchAppend0, reducedNodes);
+ calculatedAssignmentsSwitchAppend = subtract(calculatedAssignmentsSwitchAppend, addedNodes);
+ calculatedAssignmentsSwitchAppend = intersect(calculatedAssignments, calculatedAssignmentsSwitchAppend);
+
+ var calculatedPendingReduction = subtract(resolvedStable, retrievedAssignmentsSwitchReduce0);
+
+ var calculatedPendingAddition = union(resolvedStable, reducedNodes);
+ calculatedPendingAddition = intersect(calculatedAssignments, calculatedPendingAddition);
+
+ // retry preconditions should be checked inside every invoke.
+
+ Condition retryPreconditions = and(
+ revision(stablePartAssignmentsKey).eq(retrievedAssignmentsStable.revision()),
+ and(
+ revision(pendingPartAssignmentsKey).eq(retrievedAssignmentsPending.revision()),
+ and(
+ revision(switchReduceKey).eq(retrievedAssignmentsSwitchReduce.revision()),
+ revision(switchAppendKey).eq(retrievedAssignmentsSwitchAppend.revision())
+ )
+ )
+ );
+
+ Operations ops;
+
+ var rs = ByteUtils.toBytes(resolvedStable);
+ var cpa = ByteUtils.toBytes(calculatedPendingAddition);
+ var cpr = ByteUtils.toBytes(calculatedPendingReduction);
+ var casr = ByteUtils.toBytes(calculatedAssignmentsSwitchReduce);
+ var casa = ByteUtils.toBytes(calculatedAssignmentsSwitchAppend);
+
+ if (!calculatedAssignmentsSwitchAppend.isEmpty()) {
+ ops = ops(
+ put(stablePartAssignmentsKey, rs),
+ put(pendingPartAssignmentsKey, cpa),
+ put(switchReduceKey, casr),
+ put(switchAppendKey, casa)
+ );
+ } else if (!calculatedAssignmentsSwitchReduce.isEmpty()) {
+ ops = ops(
+ put(stablePartAssignmentsKey, rs),
+ put(pendingPartAssignmentsKey, cpr),
+ put(switchReduceKey, casr),
+ put(switchAppendKey, casa)
+ );
+ } else {
+ ops = ops(
+ put(pendingPartAssignmentsKey, retrievedAssignmentsPlanned.value()),
+ remove(plannedPartAssignmentsKey),
+ put(switchAppendKey, casa)
+ );
+ }
+
+ return metaStorageMgr.invoke(If.iif(retryPreconditions, ops.yield(true), ops().yield(false))).thenCompose(statementResult -> {
+ if (statementResult.getAsBoolean()) {
+ return CompletableFuture.completedFuture(null);
+ } else {
+ return onRebalanceDone(baselineNodes, partitions, replicas, newPeers, metaStorageMgr, partNum, partId);
+ }
+ });
+ });
+ }
+
+ public static List<ClusterNode> resolveClusterNodes(
+ List<PeerId> peers, byte[] pendingAssignments, byte[] stableAssignments) {
+ Map<NetworkAddress, ClusterNode> resolveRegistry = new HashMap<>();
+
+ if (pendingAssignments != null) {
+ ((List<ClusterNode>) ByteUtils.fromBytes(pendingAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
+ }
+
+ if (stableAssignments != null) {
+ ((List<ClusterNode>) ByteUtils.fromBytes(stableAssignments)).forEach(n -> resolveRegistry.put(n.address(), n));
+ }
+
+ List<ClusterNode> resolvedNodes = new ArrayList<>(peers.size());
+
+ for (PeerId p : peers) {
+ var addr = NetworkAddress.from(p.getEndpoint().getIp() + ":" + p.getEndpoint().getPort());
+
+ if (resolveRegistry.containsKey(addr)) {
+ resolvedNodes.add(resolveRegistry.get(addr));
+ } else {
+ throw new IgniteInternalException("Can't find appropriate cluster node for raft group peer: " + p);
+ }
+ }
+
+ return resolvedNodes;
+ }
}