You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sa...@apache.org on 2025/02/24 15:55:44 UTC
(ignite-3) branch main updated: IGNITE-23466 Rework DZM internals (#5092)
This is an automated email from the ASF dual-hosted git repository.
sanpwc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new bece9d1151b IGNITE-23466 Rework DZM internals (#5092)
bece9d1151b is described below
commit bece9d1151bf7ac36802ad6f8d537d0e6273d377
Author: Denis Chudov <mo...@gmail.com>
AuthorDate: Mon Feb 24 17:55:37 2025 +0200
IGNITE-23466 Rework DZM internals (#5092)
---
check-rules/spotbugs-excludes.xml | 2 +-
.../catalog/commands/AlterZoneCommand.java | 2 +-
.../catalog/commands/RenameZoneCommand.java | 2 +-
.../catalog/events/AlterZoneEventParameters.java | 18 +-
.../internal/catalog/storage/AlterZoneEntry.java | 17 +-
.../storage/CatalogEntrySerializationTest.java | 2 +-
.../ignite/internal/catalog/CatalogTestUtils.java | 11 +-
.../ignite/internal/util/io/IgniteDataInput.java | 60 +
.../ignite/internal/util/io/IgniteDataOutput.java | 40 +
...tDistributionZoneMetaStorageCompactionTest.java | 158 ++
.../ItDistributionZonesFiltersTest.java | 41 +-
...niteDistributionZoneManagerNodeRestartTest.java | 214 +--
.../rebalance/ItRebalanceDistributedTest.java | 4 +-
.../distributionzones/AugmentationSerializer.java | 66 -
.../distributionzones/DataNodesHistory.java | 158 ++
.../distributionzones/DataNodesHistoryEntry.java | 77 +
.../DataNodesHistoryMetaStorageOperation.java | 206 +++
.../distributionzones/DataNodesManager.java | 1561 ++++++++++++++++++++
.../distributionzones/DataNodesMapSerializer.java | 79 -
.../distributionzones/DistributionZoneManager.java | 1219 ++-------------
.../distributionzones/DistributionZoneTimer.java | 183 +++
.../distributionzones/DistributionZonesUtil.java | 452 +++---
.../ignite/internal/distributionzones/Node.java | 13 +-
.../distributionzones/NodeWithAttributes.java | 10 +-
.../TopologyAugmentationMapSerializer.java | 76 -
.../CausalityDataNodesEngine.java | 540 -------
.../rebalance/DistributionZoneRebalanceEngine.java | 31 +-
.../DistributionZoneRebalanceEngineV2.java | 30 +-
.../distributionzones/rebalance/RebalanceUtil.java | 3 +-
.../rebalance/ZoneRebalanceUtil.java | 23 -
.../BaseDistributionZoneManagerTest.java | 30 +-
.../DataNodesHistorySerializerTest.java | 91 ++
.../distributionzones/DataNodesManagerTest.java | 566 +++++++
.../DataNodesMapSerializerTest.java | 57 -
.../DistributionZoneManagerAlterFilterTest.java | 57 +-
...ibutionZoneManagerConfigurationChangesTest.java | 32 +-
...istributionZoneManagerScaleUpScaleDownTest.java | 839 +++--------
.../DistributionZoneTimerSerializerTest.java | 72 +
.../DistributionZonesSchedulersTest.java | 322 +---
.../TopologyAugmentationMapSerializerTest.java | 82 -
.../DistributionZoneCausalityDataNodesTest.java | 94 +-
.../DistributionZoneRebalanceEngineTest.java | 23 +-
.../DistributionZonesTestUtil.java | 162 +-
.../internal/metastorage/MetaStorageManager.java | 15 +
.../internal/metastorage/dsl/Operations.java | 11 +-
.../metastorage/impl/MetaStorageManagerImpl.java | 17 +
.../partition/replicator/fixtures/Node.java | 4 +-
.../runner/app/ItIgniteNodeRestartTest.java | 4 +-
.../org/apache/ignite/internal/app/IgniteImpl.java | 4 +-
...bstractHighAvailablePartitionsRecoveryTest.java | 66 +-
...blePartitionsRecoveryNoStaleRecoveriesTest.java | 2 +-
.../ItHighAvailablePartitionsRecoveryTest.java | 43 +-
.../ItDisasterRecoveryReconfigurationTest.java | 2 +
53 files changed, 4311 insertions(+), 3582 deletions(-)
diff --git a/check-rules/spotbugs-excludes.xml b/check-rules/spotbugs-excludes.xml
index 6fd1b28dbc5..7c22a0c17d7 100644
--- a/check-rules/spotbugs-excludes.xml
+++ b/check-rules/spotbugs-excludes.xml
@@ -242,7 +242,7 @@
<!-- Public byte array constants, not expected to be modified. -->
<Bug pattern="MS_MUTABLE_ARRAY"/>
<Class name="org.apache.ignite.internal.distributionzones.DistributionZonesUtil"/>
- <Field name="DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES"/>
+ <Field name="DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES"/>
</Match>
<Match>
<!-- I literally have no idea why these checks fail. -->
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AlterZoneCommand.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AlterZoneCommand.java
index 3fbb4d9126e..fe75ac787b4 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AlterZoneCommand.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/AlterZoneCommand.java
@@ -115,7 +115,7 @@ public class AlterZoneCommand extends AbstractZoneCommand {
CatalogZoneDescriptor descriptor = fromParamsAndPreviousValue(zone);
- return List.of(new AlterZoneEntry(descriptor));
+ return List.of(new AlterZoneEntry(descriptor, zone));
}
private CatalogZoneDescriptor fromParamsAndPreviousValue(CatalogZoneDescriptor previous) {
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/RenameZoneCommand.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/RenameZoneCommand.java
index 9fd830775a9..4555176982c 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/RenameZoneCommand.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/commands/RenameZoneCommand.java
@@ -88,7 +88,7 @@ public class RenameZoneCommand extends AbstractZoneCommand {
zone.consistencyMode()
);
- return List.of(new AlterZoneEntry(descriptor));
+ return List.of(new AlterZoneEntry(descriptor, zone));
}
private void validate() {
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/AlterZoneEventParameters.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/AlterZoneEventParameters.java
index 7fe598fa195..4c1737aabcb 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/AlterZoneEventParameters.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/events/AlterZoneEventParameters.java
@@ -26,17 +26,26 @@ public class AlterZoneEventParameters extends CatalogEventParameters {
private final CatalogZoneDescriptor zoneDescriptor;
+ private final CatalogZoneDescriptor previousDescriptor;
+
/**
* Constructor.
*
* @param causalityToken Causality token.
* @param catalogVersion Catalog version.
* @param zoneDescriptor Newly created distribution zone descriptor.
+ * @param previousDescriptor Previous distribution zone descriptor.
*/
- public AlterZoneEventParameters(long causalityToken, int catalogVersion, CatalogZoneDescriptor zoneDescriptor) {
+ public AlterZoneEventParameters(
+ long causalityToken,
+ int catalogVersion,
+ CatalogZoneDescriptor zoneDescriptor,
+ CatalogZoneDescriptor previousDescriptor
+ ) {
super(causalityToken, catalogVersion);
this.zoneDescriptor = zoneDescriptor;
+ this.previousDescriptor = previousDescriptor;
}
/**
@@ -45,4 +54,11 @@ public class AlterZoneEventParameters extends CatalogEventParameters {
public CatalogZoneDescriptor zoneDescriptor() {
return zoneDescriptor;
}
+
+ /**
+ * Gets previous distribution zone descriptor.
+ */
+ public CatalogZoneDescriptor previousDescriptor() {
+ return previousDescriptor;
+ }
}
diff --git a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/storage/AlterZoneEntry.java b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/storage/AlterZoneEntry.java
index c9bbf229af3..9cf1f930cbc 100644
--- a/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/storage/AlterZoneEntry.java
+++ b/modules/catalog/src/main/java/org/apache/ignite/internal/catalog/storage/AlterZoneEntry.java
@@ -40,13 +40,17 @@ public class AlterZoneEntry implements UpdateEntry, Fireable {
private final CatalogZoneDescriptor descriptor;
+ private final CatalogZoneDescriptor previousDescriptor;
+
/**
* Constructs the object.
*
* @param descriptor A descriptor of a zone to alter.
+ * @param previousDescriptor A previous descriptor of a zone.
*/
- public AlterZoneEntry(CatalogZoneDescriptor descriptor) {
+ public AlterZoneEntry(CatalogZoneDescriptor descriptor, CatalogZoneDescriptor previousDescriptor) {
this.descriptor = descriptor;
+ this.previousDescriptor = previousDescriptor;
}
/** Returns descriptor of a zone to alter. */
@@ -54,6 +58,11 @@ public class AlterZoneEntry implements UpdateEntry, Fireable {
return descriptor;
}
+ /** Returns previous descriptor of a zone. */
+ public CatalogZoneDescriptor previousDescriptor() {
+ return previousDescriptor;
+ }
+
@Override
public int typeId() {
return MarshallableEntryType.ALTER_ZONE.id();
@@ -66,7 +75,7 @@ public class AlterZoneEntry implements UpdateEntry, Fireable {
@Override
public CatalogEventParameters createEventParameters(long causalityToken, int catalogVersion) {
- return new AlterZoneEventParameters(causalityToken, catalogVersion, descriptor);
+ return new AlterZoneEventParameters(causalityToken, catalogVersion, descriptor, previousDescriptor);
}
@Override
@@ -97,13 +106,15 @@ public class AlterZoneEntry implements UpdateEntry, Fireable {
@Override
public AlterZoneEntry readFrom(IgniteDataInput input) throws IOException {
CatalogZoneDescriptor descriptor = CatalogZoneDescriptor.SERIALIZER.readFrom(input);
+ CatalogZoneDescriptor previousDescriptor = CatalogZoneDescriptor.SERIALIZER.readFrom(input);
- return new AlterZoneEntry(descriptor);
+ return new AlterZoneEntry(descriptor, previousDescriptor);
}
@Override
public void writeTo(AlterZoneEntry object, IgniteDataOutput output) throws IOException {
CatalogZoneDescriptor.SERIALIZER.writeTo(object.descriptor(), output);
+ CatalogZoneDescriptor.SERIALIZER.writeTo(object.previousDescriptor(), output);
}
}
}
diff --git a/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/storage/CatalogEntrySerializationTest.java b/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/storage/CatalogEntrySerializationTest.java
index 1254fd2ae6d..e0d8b348532 100644
--- a/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/storage/CatalogEntrySerializationTest.java
+++ b/modules/catalog/src/test/java/org/apache/ignite/internal/catalog/storage/CatalogEntrySerializationTest.java
@@ -255,7 +255,7 @@ public class CatalogEntrySerializationTest extends BaseIgniteAbstractTest {
private void checkAlterZoneEntry() {
CatalogStorageProfilesDescriptor profiles =
new CatalogStorageProfilesDescriptor(List.of(new CatalogStorageProfileDescriptor("default")));
- UpdateEntry entry1 = new AlterZoneEntry(newCatalogZoneDescriptor("zone1", profiles));
+ UpdateEntry entry1 = new AlterZoneEntry(newCatalogZoneDescriptor("zone1", profiles), newCatalogZoneDescriptor("zone0", profiles));
VersionedUpdate update = newVersionedUpdate(entry1, entry1);
diff --git a/modules/catalog/src/testFixtures/java/org/apache/ignite/internal/catalog/CatalogTestUtils.java b/modules/catalog/src/testFixtures/java/org/apache/ignite/internal/catalog/CatalogTestUtils.java
index d39e8c9361f..4afe9804268 100644
--- a/modules/catalog/src/testFixtures/java/org/apache/ignite/internal/catalog/CatalogTestUtils.java
+++ b/modules/catalog/src/testFixtures/java/org/apache/ignite/internal/catalog/CatalogTestUtils.java
@@ -47,6 +47,7 @@ import org.apache.ignite.internal.catalog.commands.CreateZoneCommandBuilder;
import org.apache.ignite.internal.catalog.commands.DropTableCommand;
import org.apache.ignite.internal.catalog.commands.StorageProfileParams;
import org.apache.ignite.internal.catalog.descriptors.CatalogTableDescriptor;
+import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.catalog.storage.ObjectIdGenUpdateEntry;
import org.apache.ignite.internal.catalog.storage.SnapshotEntry;
import org.apache.ignite.internal.catalog.storage.UpdateEntry;
@@ -531,21 +532,27 @@ public class CatalogTestUtils {
* Waits till default zone appears in latest version of catalog.
*
* @param manager Catalog manager to monitor.
+ * @return Default zone descriptor.
*/
- public static void awaitDefaultZoneCreation(CatalogManager manager) {
+ public static CatalogZoneDescriptor awaitDefaultZoneCreation(CatalogManager manager) {
try {
int[] versionHolder = new int[1];
+ CatalogZoneDescriptor[] catalogZoneDescriptor = new CatalogZoneDescriptor[1];
assertTrue(waitForCondition(() -> {
int latestVersion = manager.latestCatalogVersion();
versionHolder[0] = latestVersion;
- return manager.catalog(latestVersion).defaultZone() != null;
+ catalogZoneDescriptor[0] = manager.catalog(latestVersion).defaultZone();
+
+ return catalogZoneDescriptor[0] != null;
}, 5_000));
// additionally we have to wait till all listeners complete handling of event
await(manager.catalogReadyFuture(versionHolder[0]));
+
+ return catalogZoneDescriptor[0];
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataInput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataInput.java
index 93d449e2700..3392aeb5a54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataInput.java
@@ -29,7 +29,10 @@ import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Period;
import java.util.BitSet;
+import java.util.Collection;
+import java.util.Map;
import java.util.UUID;
+import java.util.function.IntFunction;
/**
* Extended data input.
@@ -418,4 +421,61 @@ public interface IgniteDataInput extends DataInput {
*/
T materialize(byte[] buffer, int offset, int length);
}
+
+ /**
+ * Reads a collection.
+ *
+ * @param collectionFunction Function that creates a collection with given initial capacity, if applicable.
+ * @param elementReader Function to read an element.
+ * @return Collection.
+ */
+ default <T, C extends Collection<T>> C readCollection(
+ IntFunction<C> collectionFunction,
+ ObjectReader<T> elementReader
+ ) throws IOException {
+ int size = readVarIntAsInt();
+
+ C collection = collectionFunction.apply(size);
+
+ for (int i = 0; i < size; i++) {
+ collection.add(elementReader.read(this));
+ }
+
+ return collection;
+ }
+
+ /**
+ * Reads a map.
+ *
+ * @param mapFunction Function that creates a map with given initial capacity, if applicable.
+ * @param keyReader Function to read a key.
+ * @param valueReader Function to read a value.
+ * @return Map.
+ */
+ default <K, C, V extends C, M extends Map<K, V>> M readMap(
+ IntFunction<M> mapFunction,
+ ObjectReader<K> keyReader,
+ ObjectReader<V> valueReader
+ ) throws IOException {
+ int size = readVarIntAsInt();
+
+ M map = mapFunction.apply(size);
+
+ for (int i = 0; i < size; i++) {
+ K key = keyReader.read(this);
+ V value = valueReader.read(this);
+
+ map.put(key, value);
+ }
+
+ return map;
+ }
+
+ /**
+ * Object reader interface.
+ */
+ @FunctionalInterface
+ interface ObjectReader<T> {
+ T read(IgniteDataInput in) throws IOException;
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataOutput.java
index befdacf0367..be94c18f20e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/IgniteDataOutput.java
@@ -29,6 +29,8 @@ import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Period;
import java.util.BitSet;
+import java.util.Collection;
+import java.util.Map;
import java.util.UUID;
/**
@@ -234,4 +236,42 @@ public interface IgniteDataOutput extends DataOutput {
* @throws IOException if something went wrong
*/
void flush() throws IOException;
+
+ /**
+ * Writes a collection.
+ *
+ * @param collection Collection.
+ * @param elementWriter Element writer.
+ */
+ default <T> void writeCollection(Collection<T> collection, ObjectWriter<T> elementWriter) throws IOException {
+ writeVarInt(collection.size());
+
+ for (T e : collection) {
+ elementWriter.write(e, this);
+ }
+ }
+
+ /**
+ * Writes a map.
+ *
+ * @param map Map.
+ * @param keyWriter Key writer.
+ * @param valWriter Value writer.
+ */
+ default <K, V> void writeMap(Map<K, V> map, ObjectWriter<K> keyWriter, ObjectWriter<V> valWriter) throws IOException {
+ writeVarInt(map.size());
+
+ for (Map.Entry<K, V> e : map.entrySet()) {
+ keyWriter.write(e.getKey(), this);
+ valWriter.write(e.getValue(), this);
+ }
+ }
+
+ /**
+ * Object writer interface.
+ */
+ @FunctionalInterface
+ interface ObjectWriter<T> {
+ void write(T obj, IgniteDataOutput out) throws IOException;
+ }
}
diff --git a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZoneMetaStorageCompactionTest.java b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZoneMetaStorageCompactionTest.java
new file mode 100644
index 00000000000..d6ba69d5ade
--- /dev/null
+++ b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZoneMetaStorageCompactionTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.stream.Collectors.toSet;
+import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl;
+import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
+import static org.apache.ignite.internal.metastorage.impl.MetaStorageCompactionTriggerConfiguration.DATA_AVAILABILITY_TIME_SYSTEM_PROPERTY_NAME;
+import static org.apache.ignite.internal.metastorage.impl.MetaStorageCompactionTriggerConfiguration.INTERVAL_SYSTEM_PROPERTY_NAME;
+import static org.apache.ignite.internal.sql.engine.util.SqlTestUtils.executeUpdate;
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
+import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow;
+import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite.InitParametersBuilder;
+import org.apache.ignite.internal.ClusterPerClassIntegrationTest;
+import org.apache.ignite.internal.app.IgniteImpl;
+import org.apache.ignite.internal.catalog.CatalogManager;
+import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.lang.ByteArray;
+import org.apache.ignite.internal.metastorage.Entry;
+import org.apache.ignite.internal.metastorage.exceptions.CompactedException;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test for case of meta storage compaction.
+ */
+public class ItDistributionZoneMetaStorageCompactionTest extends ClusterPerClassIntegrationTest {
+ private static final String ZONE_NAME = "TEST_ZONE";
+
+ @Override
+ protected int initialNodes() {
+ return 2;
+ }
+
+ @Override
+ protected void configureInitParameters(InitParametersBuilder builder) {
+ builder.clusterConfiguration(createClusterConfigWithCompactionProperties(10, 10));
+ }
+
+ private static String createClusterConfigWithCompactionProperties(long interval, long dataAvailabilityTime) {
+ return String.format(
+ "ignite.system.properties: {"
+ + "%s = \"%s\", "
+ + "%s = \"%s\""
+ + "}",
+ INTERVAL_SYSTEM_PROPERTY_NAME, interval, DATA_AVAILABILITY_TIME_SYSTEM_PROPERTY_NAME, dataAvailabilityTime
+ );
+ }
+
+ /**
+ * Tests that data nodes history is available for timestamp that matches a revision that was compacted.
+ */
+ @Test
+ public void testCompaction() throws InterruptedException {
+ String zoneSql = "create zone " + ZONE_NAME + " with partitions=1, storage_profiles='" + DEFAULT_STORAGE_PROFILE + "'"
+ + ", data_nodes_auto_adjust_scale_down=0";
+
+ CLUSTER.doInSession(0, session -> {
+ executeUpdate(zoneSql, session);
+ });
+
+ IgniteImpl ignite = unwrapIgniteImpl(CLUSTER.node(0));
+
+ HybridTimestamp beforeNodesStop = ignite.clock().now();
+
+ CatalogManager catalogManager = ignite.catalogManager();
+ CatalogZoneDescriptor zone = catalogManager.activeCatalog(beforeNodesStop.longValue())
+ .zone(ZONE_NAME);
+
+ int zoneId = zone.id();
+
+ Set<String> dataNodesBeforeNodeStop = dataNodes(ignite, zoneId, beforeNodesStop);
+
+ assertEquals(initialNodes(), dataNodesBeforeNodeStop.size());
+
+ assertTrue(waitForCondition(
+ () -> dataNodesInLocalMetaStorage(ignite, zoneId, beforeNodesStop).size() == initialNodes(),
+ 1000
+ ));
+
+ long revisionAfterCreateZone = ignite.metaStorageManager().appliedRevision();
+
+ // Put some data to increment revision once more.
+ ignite.metaStorageManager().put(new ByteArray("dummy_key"), "dummy_value".getBytes());
+
+ assertTrue(waitForCondition(() -> ignite.metaStorageManager().appliedRevision() > revisionAfterCreateZone, 1000));
+
+ CLUSTER.stopNode(1);
+
+ // Wait for data nodes adjustment.
+ assertTrue(waitForCondition(
+ () -> dataNodes(ignite, zoneId, ignite.clock().now()).size() == 1,
+ 10_000
+ ));
+
+ // Wait for meta storage to be compacted.
+ assertTrue(waitForCondition(() -> ignite.metaStorageManager().getCompactionRevisionLocally() > revisionAfterCreateZone, 1000));
+
+ // Check that old revision is not available after compaction.
+ assertThat(
+ ignite.metaStorageManager().get(zoneDataNodesHistoryKey(zoneId), revisionAfterCreateZone),
+ willThrow(CompactedException.class)
+ );
+
+ // Check that data nodes for old timestamp are still available.
+ assertEquals(dataNodesBeforeNodeStop, dataNodes(ignite, zoneId, beforeNodesStop));
+ }
+
+ private static Set<String> dataNodes(IgniteImpl ignite, int zoneId, HybridTimestamp ts) {
+ CompletableFuture<Set<String>> dataNodesBeforeStopFut = ignite
+ .distributionZoneManager()
+ .dataNodesManager()
+ .dataNodes(zoneId, ts);
+
+ assertThat(dataNodesBeforeStopFut, willCompleteSuccessfully());
+
+ return dataNodesBeforeStopFut.join();
+ }
+
+ private static Set<String> dataNodesInLocalMetaStorage(IgniteImpl ignite, int zoneId, HybridTimestamp ts) {
+ Entry e = ignite.metaStorageManager().getLocally(zoneDataNodesHistoryKey(zoneId));
+
+ if (e.empty()) {
+ return Set.of();
+ }
+
+ DataNodesHistory history = DataNodesHistorySerializer.deserialize(e.value());
+ Set<String> nodes = history.dataNodesForTimestamp(ts).dataNodes().stream()
+ .map(NodeWithAttributes::nodeName)
+ .collect(toSet());
+
+ return nodes;
+ }
+}
diff --git a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZonesFiltersTest.java b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZonesFiltersTest.java
index 24a83ed6724..03d48634881 100644
--- a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZonesFiltersTest.java
+++ b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItDistributionZonesFiltersTest.java
@@ -24,8 +24,8 @@ import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.DEFAULT_FILTER;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertValueInStorage;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeDataNodesMap;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.deserializeLatestDataNodesHistoryEntry;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.pendingPartAssignmentsKey;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.stablePartAssignmentsKey;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
@@ -42,6 +42,7 @@ import org.apache.ignite.internal.ClusterPerTestIntegrationTest;
import org.apache.ignite.internal.app.IgniteImpl;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.catalog.events.CatalogEvent;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
import org.apache.ignite.internal.lang.ByteArray;
import org.apache.ignite.internal.metastorage.Entry;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
@@ -162,13 +163,13 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(zoneId),
- (v) -> deserializeDataNodesMap(v).size(),
+ zoneDataNodesHistoryKey(zoneId),
+ DataNodesHistorySerializer::deserialize,
4,
TIMEOUT_MILLIS
);
- Entry dataNodesEntry1 = metaStorageManager.get(zoneDataNodesKey(zoneId)).get(5_000, MILLISECONDS);
+ Entry dataNodesEntry1 = metaStorageManager.get(zoneDataNodesHistoryKey(zoneId)).get(5_000, MILLISECONDS);
assertTrue(waitForCondition(() -> metaStorageManager.appliedRevision() >= dataNodesEntry1.revision(), TIMEOUT_MILLIS));
@@ -295,7 +296,7 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
node0.sql().execute(null, alterZoneSql(newFilter));
- waitDataNodeAndListenersAreHandled(metaStorageManager, 2, zoneId);
+ waitDataNodeAndListenersAreHandled(metaStorageManager, 0, zoneId);
assertValueInStorage(
metaStorageManager,
@@ -339,7 +340,7 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
int zoneId = getZoneId(node0);
- waitDataNodeAndListenersAreHandled(metaStorageManager, 2, zoneId);
+ waitDataNodeAndListenersAreHandled(metaStorageManager, 1, zoneId);
node1.sql().execute(null, createTableSql());
@@ -350,15 +351,15 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
TablePartitionId partId = new TablePartitionId(table.tableId(), 0);
// Table was created after both nodes was up, so there wasn't any rebalance.
- assertPendingAssignmentsWereNeverExist(metaStorageManager, partId);
+ assertPendingAssignmentsNeverExisted(metaStorageManager, partId);
// Stop node, that was only one, that passed the filter, so data nodes after filtering will be empty.
stopNode(1);
- waitDataNodeAndListenersAreHandled(metaStorageManager, 1, zoneId);
+ waitDataNodeAndListenersAreHandled(metaStorageManager, 0, zoneId);
// Check that pending are null, so there wasn't any rebalance.
- assertPendingAssignmentsWereNeverExist(metaStorageManager, partId);
+ assertPendingAssignmentsNeverExisted(metaStorageManager, partId);
}
@ParameterizedTest
@@ -387,7 +388,7 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
int zoneId = getZoneId(node0);
- waitDataNodeAndListenersAreHandled(metaStorageManager, 2, zoneId);
+ waitDataNodeAndListenersAreHandled(metaStorageManager, 1, zoneId);
node0.sql().execute(null, createTableSql());
@@ -398,15 +399,15 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
TablePartitionId partId = new TablePartitionId(table.tableId(), 0);
// Table was created after both nodes was up, so there wasn't any rebalance.
- assertPendingAssignmentsWereNeverExist(metaStorageManager, partId);
+ assertPendingAssignmentsNeverExisted(metaStorageManager, partId);
// Stop node, that was only one, that passed the filter, so data nodes after filtering will be empty.
stopNode(1);
- waitDataNodeAndListenersAreHandled(metaStorageManager, 1, zoneId);
+ waitDataNodeAndListenersAreHandled(metaStorageManager, 0, zoneId);
// Check that stable and pending are null, so there wasn't any rebalance.
- assertPendingAssignmentsWereNeverExist(metaStorageManager, partId);
+ assertPendingAssignmentsNeverExisted(metaStorageManager, partId);
node0.sql().execute(null, alterZoneSql(2));
@@ -426,7 +427,7 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
assertTrue(latch.await(10_000, MILLISECONDS));
// Check that stable and pending are null, so there wasn't any rebalance.
- assertPendingAssignmentsWereNeverExist(metaStorageManager, partId);
+ assertPendingAssignmentsNeverExisted(metaStorageManager, partId);
}
private static void waitDataNodeAndListenersAreHandled(
@@ -436,8 +437,8 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
) throws Exception {
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(zoneId),
- (v) -> deserializeDataNodesMap(v).size(),
+ zoneDataNodesHistoryKey(zoneId),
+ (v) -> deserializeLatestDataNodesHistoryEntry(v).size(),
expectedDataNodesSize,
TIMEOUT_MILLIS
);
@@ -453,14 +454,14 @@ public class ItDistributionZonesFiltersTest extends ClusterPerTestIntegrationTes
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(zoneId),
- (v) -> deserializeDataNodesMap(v).size(),
+ zoneDataNodesHistoryKey(zoneId),
+ (v) -> deserializeLatestDataNodesHistoryEntry(v).size(),
expectedDataNodesSize,
TIMEOUT_MILLIS
);
}
- private static void assertPendingAssignmentsWereNeverExist(
+ private static void assertPendingAssignmentsNeverExisted(
MetaStorageManager metaStorageManager,
TablePartitionId partId
) throws InterruptedException, ExecutionException {
diff --git a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItIgniteDistributionZoneManagerNodeRestartTest.java b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItIgniteDistributionZoneManagerNodeRestartTest.java
index 0414793d592..81acb90b249 100644
--- a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItIgniteDistributionZoneManagerNodeRestartTest.java
+++ b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/distributionzones/ItIgniteDistributionZoneManagerNodeRestartTest.java
@@ -17,10 +17,9 @@
package org.apache.ignite.internal.distributionzones;
-import static java.util.Collections.emptySet;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.CompletableFuture.completedFuture;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
@@ -29,16 +28,16 @@ import static org.apache.ignite.internal.distributionzones.DistributionZonesTest
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertLogicalTopologyInMetastorage;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertValueInStorage;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.createZone;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.deserializeLatestDataNodesHistoryEntry;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.getDefaultZone;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.dataNodes;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeDataNodesMap;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.toDataNodesMap;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKeyPrefix;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKeyPrefix;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLastHandledTopology;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesRecoverableStateRevision;
-import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.REBALANCE_SCHEDULER_POOL_SIZE;
+import static org.apache.ignite.internal.metastorage.dsl.OperationType.NO_OP;
import static org.apache.ignite.internal.network.utils.ClusterServiceTestUtils.defaultChannelTypeRegistry;
import static org.apache.ignite.internal.network.utils.ClusterServiceTestUtils.defaultSerializationRegistry;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.testNodeName;
@@ -71,10 +70,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
@@ -103,10 +99,11 @@ import org.apache.ignite.internal.configuration.storage.LocalFileConfigurationSt
import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
import org.apache.ignite.internal.configuration.validation.ConfigurationValidatorImpl;
import org.apache.ignite.internal.disaster.system.ClusterIdService;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.Augmentation;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.ZoneState;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.distributionzones.DataNodesManager.ZoneTimers;
import org.apache.ignite.internal.failure.FailureManager;
import org.apache.ignite.internal.failure.NoOpFailureManager;
+import org.apache.ignite.internal.hlc.ClockService;
import org.apache.ignite.internal.hlc.ClockWaiter;
import org.apache.ignite.internal.hlc.HybridClockImpl;
import org.apache.ignite.internal.hlc.TestClockService;
@@ -131,7 +128,6 @@ import org.apache.ignite.internal.security.authentication.validator.Authenticati
import org.apache.ignite.internal.testframework.ExecutorServiceExtension;
import org.apache.ignite.internal.testframework.InjectExecutorService;
import org.apache.ignite.internal.testframework.TestIgnitionManager;
-import org.apache.ignite.internal.thread.NamedThreadFactory;
import org.apache.ignite.internal.vault.VaultManager;
import org.apache.ignite.internal.worker.fixtures.NoOpCriticalWorkerRegistry;
import org.apache.ignite.network.NetworkAddress;
@@ -295,10 +291,9 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
var clockWaiter = new ClockWaiter(name, clock, commonScheduledExecutorService);
- var catalogManager = new CatalogManagerImpl(new UpdateLogImpl(metastore), new TestClockService(clock, clockWaiter));
+ ClockService clockService = new TestClockService(clock, clockWaiter);
- ScheduledExecutorService rebalanceScheduler = new ScheduledThreadPoolExecutor(REBALANCE_SCHEDULER_POOL_SIZE,
- NamedThreadFactory.create(name, "test-rebalance-scheduler", logger()));
+ var catalogManager = new CatalogManagerImpl(new UpdateLogImpl(metastore), clockService);
DistributionZoneManager distributionZoneManager = new DistributionZoneManager(
name,
@@ -306,8 +301,8 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
metastore,
logicalTopologyService,
catalogManager,
- rebalanceScheduler,
- clusterCfgMgr.configurationRegistry().getConfiguration(SystemDistributedExtensionConfiguration.KEY).system()
+ clusterCfgMgr.configurationRegistry().getConfiguration(SystemDistributedExtensionConfiguration.KEY).system(),
+ clockService
);
// Preparing the result map.
@@ -408,56 +403,6 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
assertEquals(nodeAttributesBeforeRestart, nodeAttributesAfterRestart);
}
- @ParameterizedTest(name = "defaultZone={0},consistencyMode={1}")
- @CsvSource({
- "true,",
- "false, HIGH_AVAILABILITY",
- "false, STRONG_CONSISTENCY",
- })
- public void testTopologyAugmentationMapRestoredAfterRestart(boolean defaultZone, ConsistencyMode consistencyMode) throws Exception {
- PartialNode node = startPartialNode(0);
-
- node.logicalTopology().putNode(A);
-
- String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, consistencyMode);
-
- node.logicalTopology().putNode(B);
- node.logicalTopology().putNode(C);
-
- int zoneId = getZoneId(node, zoneName);
-
- DistributionZoneManager distributionZoneManager = getDistributionZoneManager(node);
- CatalogManager catalogManager = getCatalogManager(node);
-
- assertDataNodesFromManager(
- distributionZoneManager,
- metastore::appliedRevision,
- catalogManager::latestCatalogVersion,
- zoneId,
- Set.of(A, B, C),
- TIMEOUT_MILLIS
- );
-
- ConcurrentSkipListMap<Long, Augmentation> nodeAttributesBeforeRestart =
- distributionZoneManager.zonesState().get(zoneId).topologyAugmentationMap();
-
- node.stop();
-
- node = startPartialNode(0);
-
- distributionZoneManager = getDistributionZoneManager(node);
-
- ConcurrentSkipListMap<Long, Augmentation> nodeAttributesAfterRestart =
- distributionZoneManager.zonesState().get(zoneId).topologyAugmentationMap();
-
- assertEquals(2, nodeAttributesAfterRestart.size());
-
- assertEquals(
- nodeAttributesBeforeRestart.values().stream().map(Augmentation::nodes).collect(toSet()),
- nodeAttributesAfterRestart.values().stream().map(Augmentation::nodes).collect(toSet())
- );
- }
-
@Test
public void testLogicalTopologyRestoredAfterRestart() throws Exception {
PartialNode node = startPartialNode(0);
@@ -627,7 +572,7 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
var dataNodeKey = new AtomicReference<ByteArray>();
PartialNode node = startPartialNode(0, metaStorageManager -> {
- // In this mock we catch invocation of DistributionZoneManager.initDataNodesAndTriggerKeysInMetaStorage, where condition is
+ // In this mock we catch invocation of DataNodesManager.onZoneCreate, where condition is
// based on presence of data node key in ms. After that we make this data node as a tombstone, so when logic of creation of a
// zone is run, there won't be any initialisation of data nodes keys. We try to imitate concurrent removal of a zone.
doAnswer(invocation -> {
@@ -635,7 +580,7 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
// Here we remove data nodes value for newly created zone, so it is tombstone
assertThat(
- metaStorageManager.put(dataNodeKeyForZone, DataNodesMapSerializer.serialize(toDataNodesMap(emptySet()))),
+ metaStorageManager.put(dataNodeKeyForZone, DataNodesHistorySerializer.serialize(new DataNodesHistory())),
willCompleteSuccessfully()
);
@@ -655,7 +600,7 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
byte[][] keysFromIf = iif1.cond().keys();
Optional<ByteArray> dataNodeKeyOptional = Arrays.stream(keysFromIf)
- .filter(op -> startsWith(op, zoneDataNodesKey().bytes()))
+ .filter(op -> startsWith(op, DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX.getBytes(UTF_8)))
.findFirst()
.map(ByteArray::new);
@@ -681,8 +626,8 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
assertValueInStorage(
metastore,
- zoneDataNodesKey(zoneId),
- (v) -> dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
+ zoneDataNodesHistoryKey(zoneId),
+ (v) -> dataNodes(deserializeLatestDataNodesHistoryEntry(v)).stream().map(Node::nodeName).collect(toSet()),
Set.of(A.name(), B.name(), C.name()),
TIMEOUT_MILLIS
);
@@ -762,7 +707,7 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
assertLogicalTopologyInMetastorage(Set.of(A, B), metastore);
- String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 1, 1, consistencyMode);
+ String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 100, 100, consistencyMode);
int zoneId = getZoneId(node, zoneName);
@@ -778,11 +723,6 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
TIMEOUT_MILLIS
);
- // Block scale up
- CountDownLatch scaleUpLatch = blockScaleUpTaskExecution(zoneId, distributionZoneManager);
-
- assertTrue(waitForCondition(() -> scaleUpLatch.getCount() == 1, TIMEOUT_MILLIS));
-
node.logicalTopology().putNode(C);
node.logicalTopology().removeNodes(Set.of(B));
node.logicalTopology().putNode(B);
@@ -800,24 +740,18 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
node.stop();
- scaleUpLatch.countDown();
+ log.info("Test: restarting node.");
node = startPartialNode(0);
+ alterZone(node, zoneName, 0, 0, null);
+
distributionZoneManager = getDistributionZoneManager(node);
catalogManager = getCatalogManager(node);
assertDataNodesFromManager(distributionZoneManager, metastore::appliedRevision, catalogManager::latestCatalogVersion, zoneId,
Set.of(A, B, C), TIMEOUT_MILLIS);
-
- assertValueInStorage(
- metastore,
- zoneDataNodesKey(zoneId),
- (v) -> dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
- Set.of(A.name(), B.name(), C.name()),
- TIMEOUT_MILLIS
- );
}
@ParameterizedTest(name = "defaultZone={0},consistencyMode={1}")
@@ -837,14 +771,18 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
DistributionZoneManager distributionZoneManager = getDistributionZoneManager(node);
CatalogManager catalogManager = getCatalogManager(node);
- String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 1, 1, consistencyMode);
-
- int zoneId = getZoneId(node, zoneName);
+ assertDataNodesFromManager(
+ distributionZoneManager,
+ metastore::appliedRevision,
+ catalogManager::latestCatalogVersion,
+ 0,
+ Set.of(A, B),
+ TIMEOUT_MILLIS
+ );
- // Block scale down
- CountDownLatch scaleDownLatch = blockScaleDownTaskExecution(zoneId, distributionZoneManager);
+ String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 100, 100, consistencyMode);
- assertTrue(waitForCondition(() -> scaleDownLatch.getCount() == 1, TIMEOUT_MILLIS));
+ int zoneId = getZoneId(node, zoneName);
node.logicalTopology().removeNodes(Set.of(B));
node.logicalTopology().putNode(C);
@@ -860,27 +798,22 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
node.stop();
- scaleDownLatch.countDown();
+ log.info("Test: restarting node.");
node = startPartialNode(0);
+ alterZone(node, zoneName, 0, 0, null);
+
catalogManager = getCatalogManager(node);
assertDataNodesFromManager(
getDistributionZoneManager(node),
- metastore::appliedRevision, catalogManager::latestCatalogVersion,
+ metastore::appliedRevision,
+ catalogManager::latestCatalogVersion,
zoneId,
Set.of(A, C),
TIMEOUT_MILLIS
);
-
- assertValueInStorage(
- metastore,
- zoneDataNodesKey(zoneId),
- (v) -> dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
- Set.of(A.name(), C.name()),
- TIMEOUT_MILLIS
- );
}
private static String createZoneOrAlterDefaultZone(
@@ -900,15 +833,15 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
alterZone(node, zoneName, scaleUp, scaleDown, null);
- ZoneState zoneState = getDistributionZoneManager(node).zonesState().get(defaultZone.id());
+ ZoneTimers zoneTimers = getDistributionZoneManager(node).dataNodesManager().zoneTimers(defaultZone.id());
// This is needed because we want to wait for the end of scale up/down triggered by altering delays.
- if (zoneState.scaleUpTask() != null) {
- assertTrue(waitForCondition(() -> zoneState.scaleUpTask().isDone(), TIMEOUT_MILLIS));
+ if (zoneTimers.scaleUp.taskIsScheduled()) {
+ assertTrue(waitForCondition(zoneTimers.scaleUp::taskIsDone, TIMEOUT_MILLIS));
}
- if (zoneState.scaleDownTask() != null) {
- assertTrue(waitForCondition(() -> zoneState.scaleDownTask().isDone(), TIMEOUT_MILLIS));
+ if (zoneTimers.scaleDown.taskIsScheduled()) {
+ assertTrue(waitForCondition(zoneTimers.scaleDown::taskIsDone, TIMEOUT_MILLIS));
}
} else {
zoneName = ZONE_NAME;
@@ -923,16 +856,16 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
doThrow(new RuntimeException("Expected")).when(metaStorageManager).invoke(argThat(iif -> {
If iif1 = MetaStorageWriteHandler.toIf(iif);
- byte[] keyScaleUpBytes = zoneScaleUpChangeTriggerKeyPrefix().bytes();
- byte[] keyScaleDownBytes = zoneScaleDownChangeTriggerKeyPrefix().bytes();
+ byte[] keyScaleUpBytes = DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX.getBytes(UTF_8);
+ byte[] keyScaleDownBytes = DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX.getBytes(UTF_8);
byte[] keyGlobalStateBytes = zonesRecoverableStateRevision().bytes();
boolean isScaleUpKey = iif1.andThen().update().operations().stream()
- .anyMatch(op -> startsWith(toByteArray(op.key()), keyScaleUpBytes));
+ .anyMatch(op -> op.type() != NO_OP && startsWith(toByteArray(op.key()), keyScaleUpBytes));
boolean isScaleDownKey = iif1.andThen().update().operations().stream()
- .anyMatch(op -> startsWith(toByteArray(op.key()), keyScaleDownBytes));
+ .anyMatch(op -> op.type() != NO_OP && startsWith(toByteArray(op.key()), keyScaleDownBytes));
boolean isGlobalStateChangeKey = iif1.andThen().update().operations().stream()
- .anyMatch(op -> startsWith(toByteArray(op.key()), keyGlobalStateBytes));
+ .anyMatch(op -> op.type() != NO_OP && startsWith(toByteArray(op.key()), keyGlobalStateBytes));
return isScaleUpKey && startScaleUpBlocking
|| isScaleDownKey && startScaleDownBlocking
@@ -961,7 +894,7 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
}
private static int getDefaultZoneId(PartialNode node) {
- return DistributionZonesTestUtil.getDefaultZone(getCatalogManager(node), node.clock().nowLong()).id();
+ return getDefaultZone(getCatalogManager(node), node.clock().nowLong()).id();
}
private static void alterZone(
@@ -974,55 +907,6 @@ public class ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
DistributionZonesTestUtil.alterZone(getCatalogManager(node), zoneName, scaleUp, scaleDown, filter);
}
- /**
- * Schedule a scale up task which block execution of another scale up tasks.
- * Note that because of the executor of scale up/scale down tasks is single-threaded, this
- * method also blocks scale down tasks execution.
- *
- * @return Latch to unblock execution of scale up tasks.
- */
- private static CountDownLatch blockScaleUpTaskExecution(int zoneId, DistributionZoneManager distributionZoneManager) {
- CountDownLatch scaleUpLatch = new CountDownLatch(2);
-
- Runnable dummyScaleUpTask = () -> {
- try {
- scaleUpLatch.countDown();
- scaleUpLatch.await(2 * TIMEOUT_MILLIS, MILLISECONDS);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
- distributionZoneManager.zonesState().get(zoneId)
- .rescheduleScaleUp(IMMEDIATE_TIMER_VALUE, dummyScaleUpTask, zoneId);
-
- return scaleUpLatch;
- }
-
- /**
- * Schedule a scale down task which block execution of another scale down tasks.
- * Note that because of the executor of scale up/scale down tasks is single-threaded, this
- * method also blocks scale up tasks execution.
- *
- * @return Latch to unblock execution of scale down tasks.
- */
- private static CountDownLatch blockScaleDownTaskExecution(int zoneId, DistributionZoneManager distributionZoneManager) {
- CountDownLatch scaleDownLatch = new CountDownLatch(2);
-
- Runnable dummyScaleDownTask = () -> {
- try {
- scaleDownLatch.countDown();
- scaleDownLatch.await(TIMEOUT_MILLIS, MILLISECONDS);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- distributionZoneManager.zonesState().get(zoneId)
- .rescheduleScaleDown(IMMEDIATE_TIMER_VALUE, dummyScaleDownTask, zoneId);
-
- return scaleDownLatch;
- }
-
private static void awaitDefaultZoneCreation(PartialNode node) {
CatalogManager manager = findComponent(node.startedComponents(), CatalogManager.class);
diff --git a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java
index 4698a42f7ec..70845a790e8 100644
--- a/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java
+++ b/modules/distribution-zones/src/integrationTest/java/org/apache/ignite/internal/rebalance/ItRebalanceDistributedTest.java
@@ -1466,8 +1466,8 @@ public class ItRebalanceDistributedTest extends BaseIgniteAbstractTest {
metaStorageManager,
logicalTopologyService,
catalogManager,
- rebalanceScheduler,
- systemDistributedConfiguration
+ systemDistributedConfiguration,
+ clockService
);
StorageUpdateConfiguration storageUpdateConfiguration = clusterConfigRegistry
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/AugmentationSerializer.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/AugmentationSerializer.java
deleted file mode 100644
index 057802f2e3b..00000000000
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/AugmentationSerializer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.distributionzones;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.Augmentation;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.io.IgniteDataInput;
-import org.apache.ignite.internal.util.io.IgniteDataOutput;
-import org.apache.ignite.internal.versioned.VersionedSerializer;
-
-/**
- * {@link VersionedSerializer} for {@link Augmentation} instances.
- */
-public class AugmentationSerializer extends VersionedSerializer<Augmentation> {
- /** Serializer instance. */
- public static final AugmentationSerializer INSTANCE = new AugmentationSerializer();
-
- private final NodeSerializer nodeSerializer = NodeSerializer.INSTANCE;
-
- @Override
- protected void writeExternalData(Augmentation augmentation, IgniteDataOutput out) throws IOException {
- out.writeVarInt(augmentation.nodes().size());
- for (Node node : augmentation.nodes()) {
- nodeSerializer.writeExternal(node, out);
- }
-
- out.writeBoolean(augmentation.addition());
- }
-
- @Override
- protected Augmentation readExternalData(byte protoVer, IgniteDataInput in) throws IOException {
- Set<Node> nodes = readNodes(in);
- boolean addition = in.readBoolean();
-
- return new Augmentation(nodes, addition);
- }
-
- private Set<Node> readNodes(IgniteDataInput in) throws IOException {
- int length = in.readVarIntAsInt();
-
- Set<Node> nodes = new HashSet<>(IgniteUtils.capacity(length));
- for (int i = 0; i < length; i++) {
- nodes.add(nodeSerializer.readExternal(in));
- }
-
- return nodes;
- }
-}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistory.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistory.java
new file mode 100644
index 00000000000..1653cac4bee
--- /dev/null
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistory.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.Collections.emptySet;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.util.io.IgniteDataInput;
+import org.apache.ignite.internal.util.io.IgniteDataOutput;
+import org.apache.ignite.internal.versioned.VersionedSerialization;
+import org.apache.ignite.internal.versioned.VersionedSerializer;
+
+/**
+ * Data nodes history. Is actually a map of timestamps to sets of nodes with their attributes.
+ */
+public class DataNodesHistory {
+ private final NavigableMap<HybridTimestamp, Set<NodeWithAttributes>> history;
+
+ public DataNodesHistory() {
+ this(new TreeMap<>());
+ }
+
+ private DataNodesHistory(NavigableMap<HybridTimestamp, Set<NodeWithAttributes>> history) {
+ this.history = history;
+ }
+
+ /**
+ * Copies existing history and adds a new history entry.
+ *
+ * @param timestamp Timestamp.
+ * @param nodes Nodes.
+ * @return New data nodes history.
+ */
+ public DataNodesHistory addHistoryEntry(HybridTimestamp timestamp, Set<NodeWithAttributes> nodes) {
+ DataNodesHistory dataNodesHistory = new DataNodesHistory(new TreeMap<>(this.history));
+ dataNodesHistory.history.put(timestamp, nodes);
+ return dataNodesHistory;
+ }
+
+ /**
+ * Checks that the exact timestamp is present in history.
+ *
+ * @param timestamp Timestamp.
+ * @return {@code true} if the exact timestamp is present in history.
+ */
+ public boolean entryIsPresentAtExactTimestamp(HybridTimestamp timestamp) {
+ return history.containsKey(timestamp);
+ }
+
+ /**
+ * Checks that the history is empty.
+ *
+ * @return {@code true} if history is empty.
+ */
+ public boolean isEmpty() {
+ return history.isEmpty();
+ }
+
+ /**
+ * Returns data nodes for timestamp, or empty set.
+ *
+ * @param timestamp Timestamp.
+ * @return Data nodes for timestamp.
+ */
+ public DataNodesHistoryEntry dataNodesForTimestamp(HybridTimestamp timestamp) {
+ Map.Entry<HybridTimestamp, Set<NodeWithAttributes>> entry = history.floorEntry(timestamp);
+
+ if (entry == null) {
+ return new DataNodesHistoryEntry(HybridTimestamp.MIN_VALUE, emptySet());
+ }
+
+ return new DataNodesHistoryEntry(entry.getKey(), entry.getValue());
+ }
+
+ @Override
+ public String toString() {
+ return "DataNodesHistory [history=" + history + "].";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DataNodesHistory history1 = (DataNodesHistory) o;
+ return Objects.equals(history, history1.history);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(history);
+ }
+
+ /**
+ * Data nodes history serializer.
+ */
+ public static class DataNodesHistorySerializer extends VersionedSerializer<DataNodesHistory> {
+ private static final DataNodesHistorySerializer INSTANCE = new DataNodesHistorySerializer();
+
+ @Override
+ protected void writeExternalData(DataNodesHistory object, IgniteDataOutput out) throws IOException {
+ // Using new TreeSet() because the order of elements is important: the serialized representation is compared during
+ // MetaStorageManager#invoke().
+ out.writeMap(
+ object.history,
+ (k, out0) -> out0.writeLong(k.longValue()),
+ (v, out0) -> out0.writeCollection(new TreeSet<>(v), NodeWithAttributesSerializer.INSTANCE::writeExternal)
+ );
+ }
+
+ @Override
+ protected DataNodesHistory readExternalData(byte protoVer, IgniteDataInput in) throws IOException {
+ NavigableMap<HybridTimestamp, Set<NodeWithAttributes>> history = in.readMap(
+ unused -> new TreeMap<>(),
+ in0 -> HybridTimestamp.hybridTimestamp(in0.readLong()),
+ in0 -> in0.readCollection(
+ unused -> new TreeSet<>(),
+ NodeWithAttributesSerializer.INSTANCE::readExternal
+ )
+ );
+
+ return new DataNodesHistory(history);
+ }
+
+ public static byte[] serialize(DataNodesHistory dataNodesHistory) {
+ return VersionedSerialization.toBytes(dataNodesHistory, INSTANCE);
+ }
+
+ public static DataNodesHistory deserialize(byte[] bytes) {
+ return VersionedSerialization.fromBytes(bytes, INSTANCE);
+ }
+ }
+}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistoryEntry.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistoryEntry.java
new file mode 100644
index 00000000000..4e90b08772d
--- /dev/null
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistoryEntry.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import java.util.Objects;
+import java.util.Set;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+
+/**
+ * Class representing a single entry in the history of data nodes.
+ */
+public class DataNodesHistoryEntry {
+ private final HybridTimestamp timestamp;
+
+ private final Set<NodeWithAttributes> dataNodes;
+
+ /**
+ * Constructor.
+ *
+ * @param timestamp Timestamp of the history entry.
+ * @param dataNodes Set of data nodes at the time of the history entry.
+ */
+ public DataNodesHistoryEntry(HybridTimestamp timestamp, Set<NodeWithAttributes> dataNodes) {
+ this.timestamp = timestamp;
+ this.dataNodes = dataNodes;
+ }
+
+ /**
+ * Timestamp of the history entry.
+ *
+ * @return Timestamp of the history entry.
+ */
+ public HybridTimestamp timestamp() {
+ return timestamp;
+ }
+
+ /**
+ * Set of data nodes at the time of the history entry.
+ *
+ * @return Set of data nodes at the time of the history entry.
+ */
+ public Set<NodeWithAttributes> dataNodes() {
+ return dataNodes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DataNodesHistoryEntry that = (DataNodesHistoryEntry) o;
+ return Objects.equals(timestamp, that.timestamp) && Objects.equals(dataNodes, that.dataNodes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestamp, dataNodes);
+ }
+}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistoryMetaStorageOperation.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistoryMetaStorageOperation.java
new file mode 100644
index 00000000000..42abbf14225
--- /dev/null
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesHistoryMetaStorageOperation.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.Collections.unmodifiableList;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.nodeNames;
+import static org.apache.ignite.internal.metastorage.dsl.Operations.ops;
+import static org.apache.ignite.internal.metastorage.dsl.Statements.iif;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.metastorage.dsl.Condition;
+import org.apache.ignite.internal.metastorage.dsl.Iif;
+import org.apache.ignite.internal.metastorage.dsl.Operation;
+import org.apache.ignite.internal.metastorage.dsl.Operations;
+import org.jetbrains.annotations.Nullable;
+
+class DataNodesHistoryMetaStorageOperation {
+ private final Condition condition;
+ private final List<Operation> operations;
+ private final String operationName;
+ private final int zoneId;
+ private final DataNodesHistory currentDataNodesHistory;
+ private final HybridTimestamp currentTimestamp;
+ private final HybridTimestamp historyEntryTimestamp;
+ private final Set<NodeWithAttributes> historyEntryNodes;
+ private final boolean addMandatoryEntry;
+
+ @Nullable
+ private final DistributionZoneTimer scaleUpTimer;
+
+ @Nullable
+ private final DistributionZoneTimer scaleDownTimer;
+
+ private DataNodesHistoryMetaStorageOperation(
+ Condition condition,
+ List<Operation> operations,
+ String operationName,
+ int zoneId,
+ DataNodesHistory currentDataNodesHistory,
+ HybridTimestamp currentTimestamp,
+ HybridTimestamp historyEntryTimestamp,
+ Set<NodeWithAttributes> historyEntryNodes,
+ boolean addMandatoryEntry,
+ @Nullable DistributionZoneTimer scaleUpTimer,
+ @Nullable DistributionZoneTimer scaleDownTimer
+ ) {
+ this.condition = condition;
+ this.operations = operations;
+ this.operationName = operationName;
+ this.zoneId = zoneId;
+ this.currentDataNodesHistory = currentDataNodesHistory;
+ this.currentTimestamp = currentTimestamp;
+ this.historyEntryTimestamp = historyEntryTimestamp;
+ this.historyEntryNodes = historyEntryNodes;
+ this.addMandatoryEntry = addMandatoryEntry;
+ this.scaleUpTimer = scaleUpTimer;
+ this.scaleDownTimer = scaleDownTimer;
+ }
+
+ static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Forms a message for the log record about the history entry update. It also specifies whether the new history entry was
+ * actually added; it is added if the history is empty, the nodes are different from the latest history entry or the
+ * {@code addMandatoryEntry} parameter is true.
+ *
+ * @return Log message.
+ */
+ String successLogMessage() {
+ Set<NodeWithAttributes> latestNodesWritten = currentDataNodesHistory.dataNodesForTimestamp(HybridTimestamp.MAX_VALUE).dataNodes();
+ Set<NodeWithAttributes> nodes = historyEntryNodes;
+ boolean historyEntryAdded = addMandatoryEntry || currentDataNodesHistory.isEmpty() || !nodes.equals(latestNodesWritten);
+
+ return "Updated data nodes on " + operationName + ", "
+ + (historyEntryAdded ? "added history entry" : "history entry not added")
+ + " [zoneId=" + zoneId + ", currentTimestamp=" + currentTimestamp
+ + (historyEntryAdded ? ", historyEntryTimestamp=" + historyEntryTimestamp + ", nodes=" + nodeNames(nodes) : "")
+ + (historyEntryAdded ? "" : ", latestNodesWritten=" + nodeNames(latestNodesWritten))
+ + "]"
+ + (scaleUpTimer == null ? "" : ", scaleUpTimer=" + scaleUpTimer)
+ + (scaleDownTimer == null ? "" : ", scaleDownTimer=" + scaleDownTimer)
+ + "].";
+ }
+
+ String failureLogMessage() {
+ return "Failed to update data nodes history and timers on " + operationName + " [timestamp="
+ + currentTimestamp + "].";
+ }
+
+ public Iif operation() {
+ return iif(condition, new Operations(operations).yield(true), ops().yield(false));
+ }
+
+ static class Builder {
+ private Condition condition;
+ private List<Operation> operations;
+ private String operationName;
+ private int zoneId = -1;
+ private DataNodesHistory currentDataNodesHistory;
+ private HybridTimestamp currentTimestamp;
+ private HybridTimestamp historyEntryTimestamp;
+ private Set<NodeWithAttributes> historyEntryNodes;
+ private boolean addMandatoryEntry;
+ private DistributionZoneTimer scaleUpTimer;
+ private DistributionZoneTimer scaleDownTimer;
+
+ Builder condition(Condition condition) {
+ this.condition = condition;
+ return this;
+ }
+
+ Builder operations(List<Operation> operations) {
+ this.operations = operations;
+ return this;
+ }
+
+ Builder operationName(String operationName) {
+ this.operationName = operationName;
+ return this;
+ }
+
+ Builder zoneId(int zoneId) {
+ this.zoneId = zoneId;
+ return this;
+ }
+
+ Builder currentDataNodesHistory(DataNodesHistory currentDataNodesHistory) {
+ this.currentDataNodesHistory = currentDataNodesHistory;
+ return this;
+ }
+
+ Builder currentTimestamp(HybridTimestamp currentTimestamp) {
+ this.currentTimestamp = currentTimestamp;
+ return this;
+ }
+
+ Builder historyEntryTimestamp(HybridTimestamp historyEntryTimestamp) {
+ this.historyEntryTimestamp = historyEntryTimestamp;
+ return this;
+ }
+
+ Builder historyEntryNodes(Set<NodeWithAttributes> historyEntryNodes) {
+ this.historyEntryNodes = historyEntryNodes;
+ return this;
+ }
+
+ Builder scaleUpTimer(@Nullable DistributionZoneTimer scaleUpTimer) {
+ this.scaleUpTimer = scaleUpTimer;
+ return this;
+ }
+
+ Builder scaleDownTimer(@Nullable DistributionZoneTimer scaleDownTimer) {
+ this.scaleDownTimer = scaleDownTimer;
+ return this;
+ }
+
+ Builder addMandatoryEntry(boolean addMandatoryEntry) {
+ this.addMandatoryEntry = addMandatoryEntry;
+ return this;
+ }
+
+ DataNodesHistoryMetaStorageOperation build() {
+ assert zoneId >= 0 : "Zone id is not set.";
+ assert condition != null : "Condition is not set, zoneId=" + zoneId;
+ assert operations != null : "Meta storage operations are not set, zoneId=" + zoneId;
+ assert operationName != null : "Operation name is not set, zoneId=" + zoneId;
+ assert currentDataNodesHistory != null : "Current data nodes history is not set, zoneId=" + zoneId;
+ assert currentTimestamp != null : "Current timestamp is not set, zoneId=" + zoneId;
+ assert historyEntryTimestamp != null : "History entry timestamp is not set, zoneId=" + zoneId;
+ assert historyEntryNodes != null : "History entry nodes are not set, zoneId=" + zoneId;
+
+ return new DataNodesHistoryMetaStorageOperation(
+ condition,
+ unmodifiableList(operations),
+ operationName,
+ zoneId,
+ currentDataNodesHistory,
+ currentTimestamp,
+ historyEntryTimestamp,
+ historyEntryNodes,
+ addMandatoryEntry,
+ scaleUpTimer,
+ scaleDownTimer
+ );
+ }
+ }
+}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesManager.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesManager.java
new file mode 100644
index 00000000000..c9d9a9f7c56
--- /dev/null
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesManager.java
@@ -0,0 +1,1561 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.lang.Math.max;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptySet;
+import static java.util.concurrent.CompletableFuture.completedFuture;
+import static java.util.concurrent.CompletableFuture.failedFuture;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.stream.Collectors.toSet;
+import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
+import static org.apache.ignite.internal.catalog.descriptors.ConsistencyMode.HIGH_AVAILABILITY;
+import static org.apache.ignite.internal.distributionzones.DistributionZoneTimer.DEFAULT_TIMER;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX_BYTES;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX_BYTES;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.createZoneManagerExecutor;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.dataNodeHistoryContextFromValues;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeLogicalTopologySet;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.filterDataNodes;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.nodeNames;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryPrefix;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonePartitionResetTimerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownTimerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownTimerPrefix;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpTimerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpTimerPrefix;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
+import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.extractZoneId;
+import static org.apache.ignite.internal.lang.IgniteStringFormatter.format;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.and;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.exists;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.notExists;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.notTombstone;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.or;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.value;
+import static org.apache.ignite.internal.metastorage.dsl.Operations.noop;
+import static org.apache.ignite.internal.metastorage.dsl.Operations.ops;
+import static org.apache.ignite.internal.metastorage.dsl.Operations.put;
+import static org.apache.ignite.internal.metastorage.dsl.Operations.remove;
+import static org.apache.ignite.internal.metastorage.dsl.Statements.iif;
+import static org.apache.ignite.internal.util.CollectionUtils.union;
+import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
+import static org.apache.ignite.internal.util.IgniteUtils.inBusyLock;
+import static org.apache.ignite.internal.util.IgniteUtils.inBusyLockAsync;
+import static org.apache.ignite.internal.util.IgniteUtils.shutdownAndAwaitTermination;
+import static org.apache.ignite.internal.util.IgniteUtils.startsWith;
+import static org.apache.ignite.lang.ErrorGroups.Common.NODE_STOPPING_ERR;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledFuture;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.IntSupplier;
+import org.apache.ignite.internal.catalog.CatalogManager;
+import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.distributionzones.DistributionZoneTimer.DistributionZoneTimerSerializer;
+import org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DataNodesHistoryContext;
+import org.apache.ignite.internal.distributionzones.exception.DistributionZoneNotFoundException;
+import org.apache.ignite.internal.hlc.ClockService;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.lang.ByteArray;
+import org.apache.ignite.internal.lang.IgniteInternalException;
+import org.apache.ignite.internal.lang.NodeStoppingException;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.metastorage.Entry;
+import org.apache.ignite.internal.metastorage.EntryEvent;
+import org.apache.ignite.internal.metastorage.MetaStorageManager;
+import org.apache.ignite.internal.metastorage.WatchEvent;
+import org.apache.ignite.internal.metastorage.WatchListener;
+import org.apache.ignite.internal.metastorage.dsl.Condition;
+import org.apache.ignite.internal.metastorage.dsl.Iif;
+import org.apache.ignite.internal.metastorage.dsl.Operation;
+import org.apache.ignite.internal.metastorage.dsl.StatementResult;
+import org.apache.ignite.internal.metastorage.dsl.Update;
+import org.apache.ignite.internal.thread.NamedThreadFactory;
+import org.apache.ignite.internal.thread.StripedScheduledThreadPoolExecutor;
+import org.apache.ignite.internal.util.IgniteSpinBusyLock;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
+import org.jetbrains.annotations.VisibleForTesting;
+
+/**
+ * Manager for data nodes of distribution zones.
+ * <br>
+ * It is used by {@link DistributionZoneManager} to calculate data nodes, see {@link #dataNodes(int, HybridTimestamp)}.
+ * Data nodes are stored in meta storage as {@link DataNodesHistory} for each zone, also for each zone there are scale up and
+ * scale down timers, stored as {@link DistributionZoneTimer}. Partition reset timer is calculated on the node recovery according
+ * to the scale down timer state, see {@link #restorePartitionResetTimer(int, DistributionZoneTimer, long)}.
+ * <br>
+ * Data nodes history is appended on topology changes, on zone filter changes and on zone auto adjust alterations (i.e. alterations of
+ * scale up and scale down timer configuration), see {@link #onTopologyChange}, {@link #onZoneFilterChange} and
+ * {@link #onAutoAdjustAlteration} methods respectively.
+ */
+public class DataNodesManager {
+ /** The logger. */
+ private static final IgniteLogger LOG = Loggers.forClass(DataNodesManager.class);
+
+ private static final int MAX_ATTEMPTS_ON_RETRY = 100;
+
+ private final MetaStorageManager metaStorageManager;
+
+ private final CatalogManager catalogManager;
+
+ private final ClockService clockService;
+
+ /** External busy lock. */
+ private final IgniteSpinBusyLock busyLock;
+
+ /**
+ * Map with zone id as a key and set of zone timers as a value.
+ */
+ private final Map<Integer, ZoneTimers> zoneTimers = new ConcurrentHashMap<>();
+
+ /** Executor for scheduling tasks for scale up and scale down processes. */
+ private final StripedScheduledThreadPoolExecutor executor;
+
+ private final String localNodeName;
+
+ private final WatchListener scaleUpTimerPrefixListener;
+
+ private final WatchListener scaleDownTimerPrefixListener;
+
+ private final WatchListener dataNodesListener;
+
+ private final Map<Integer, DataNodesHistory> dataNodesHistoryVolatile = new ConcurrentHashMap<>();
+
+ private final BiConsumer<Long, Integer> partitionResetClosure;
+
+ private final IntSupplier partitionDistributionResetTimeoutSupplier;
+
+ /**
+ * Constructor.
+ *
+ * @param nodeName Local node name.
+ * @param busyLock External busy lock.
+ * @param metaStorageManager Meta storage manager.
+ * @param catalogManager Catalog manager.
+ * @param clockService Clock service.
+ * @param partitionResetClosure Closure to reset partitions.
+ * @param partitionDistributionResetTimeoutSupplier Supplier for partition distribution reset timeout.
+ */
+ public DataNodesManager(
+ String nodeName,
+ IgniteSpinBusyLock busyLock,
+ MetaStorageManager metaStorageManager,
+ CatalogManager catalogManager,
+ ClockService clockService,
+ BiConsumer<Long, Integer> partitionResetClosure,
+ IntSupplier partitionDistributionResetTimeoutSupplier
+ ) {
+ this.metaStorageManager = metaStorageManager;
+ this.catalogManager = catalogManager;
+ this.clockService = clockService;
+ this.localNodeName = nodeName;
+ this.partitionResetClosure = partitionResetClosure;
+ this.partitionDistributionResetTimeoutSupplier = partitionDistributionResetTimeoutSupplier;
+ this.busyLock = busyLock;
+
+ executor = createZoneManagerExecutor(
+ Math.min(Runtime.getRuntime().availableProcessors() * 3, 20),
+ NamedThreadFactory.create(nodeName, "dst-zones-scheduler", LOG)
+ );
+
+ scaleUpTimerPrefixListener = createScaleUpTimerPrefixListener();
+ scaleDownTimerPrefixListener = createScaleDownTimerPrefixListener();
+ dataNodesListener = createDataNodesListener();
+ }
+
+ CompletableFuture<Void> startAsync(
+ Collection<CatalogZoneDescriptor> knownZones,
+ long recoveryRevision
+ ) {
+ metaStorageManager.registerPrefixWatch(zoneScaleUpTimerPrefix(), scaleUpTimerPrefixListener);
+ metaStorageManager.registerPrefixWatch(zoneScaleDownTimerPrefix(), scaleDownTimerPrefixListener);
+ metaStorageManager.registerPrefixWatch(zoneDataNodesHistoryPrefix(), dataNodesListener);
+
+ if (knownZones.isEmpty()) {
+ return nullCompletedFuture();
+ }
+
+ Set<ByteArray> allKeys = new HashSet<>();
+ Map<Integer, CatalogZoneDescriptor> descriptors = new HashMap<>();
+
+ for (CatalogZoneDescriptor zone : knownZones) {
+ allKeys.add(zoneDataNodesHistoryKey(zone.id()));
+ allKeys.add(zoneScaleUpTimerKey(zone.id()));
+ allKeys.add(zoneScaleDownTimerKey(zone.id()));
+ allKeys.add(zonePartitionResetTimerKey(zone.id()));
+
+ descriptors.put(zone.id(), zone);
+ }
+
+ return metaStorageManager.getAll(allKeys)
+ .thenAccept(entriesMap -> {
+ for (CatalogZoneDescriptor zone : descriptors.values()) {
+ Entry historyEntry = entriesMap.get(zoneDataNodesHistoryKey(zone.id()));
+ Entry scaleUpEntry = entriesMap.get(zoneScaleUpTimerKey(zone.id()));
+ Entry scaleDownEntry = entriesMap.get(zoneScaleDownTimerKey(zone.id()));
+ Entry partitionResetEntry = entriesMap.get(zonePartitionResetTimerKey(zone.id()));
+
+ if (missingEntry(historyEntry)) {
+ // Not critical because if we have no history in this map, we look into meta storage.
+ LOG.warn("Couldn't recover data nodes history for zone [id={}, historyEntry={}].", zone.id(), historyEntry);
+
+ continue;
+ }
+
+ if (missingEntry(scaleUpEntry) || missingEntry(scaleDownEntry) || missingEntry(partitionResetEntry)) {
+ throw new AssertionError(format("Couldn't recover timers for zone [id={}, name={}, scaleUpEntry={}, "
+ + "scaleDownEntry={}, partitionResetEntry={}", zone.id(), zone.name(), scaleUpEntry,
+ scaleDownEntry, partitionResetEntry));
+ }
+
+ DataNodesHistory history = DataNodesHistorySerializer.deserialize(historyEntry.value());
+ dataNodesHistoryVolatile.put(zone.id(), history);
+
+ DistributionZoneTimer scaleUpTimer = DistributionZoneTimerSerializer.deserialize(scaleUpEntry.value());
+ DistributionZoneTimer scaleDownTimer = DistributionZoneTimerSerializer.deserialize(scaleDownEntry.value());
+
+ onScaleUpTimerChange(zone, scaleUpTimer);
+ onScaleDownTimerChange(zone, scaleDownTimer);
+ restorePartitionResetTimer(zone.id(), scaleDownTimer, recoveryRevision);
+ }
+ });
+ }
+
+ private static boolean missingEntry(Entry e) {
+ return e.empty() || e.tombstone();
+ }
+
+ void stop() {
+ zoneTimers.forEach((k, zt) -> zt.stopAllTimers());
+
+ shutdownAndAwaitTermination(executor, 10, SECONDS);
+ }
+
+ /**
+ * Recalculates data nodes on topology changes, modifies scale up and scale down timers and appends data nodes history in meta storage.
+ * It takes the current data nodes according to timestamp, compares them with the new topology and calculates added nodes and
+ * removed nodes, also compares with the old topology and calculates added nodes comparing to the old topology (if they are not empty
+ * then the history entry will be always added).
+ * <br>
+ * Added nodes and removed nodes are added to scale up and scale down timers respectively. Their time to trigger is refreshed
+ * according to current zone descriptor. If their time to trigger is less than or equal to the current timestamp (including the cases
+ * when auto adjust timeout is immediate), then they are applied automatically to the new history entry, in this cases their value
+ * in meta storage is set to {@link DistributionZoneTimer#DEFAULT_TIMER}.
+ * <br>
+ * For example:
+ * <ul>
+ * <li>there are nodes A, B in the latest data nodes history entry, its timestamp is 1;</li>
+ * <li>scale up auto adjust is 5, scale down auto adjust is 100 (let's say that time units for auto adjust are the same as used
+ * for timestamps);</li>
+ * <li>node A leaves, node C joins at timestamp 10;</li>
+ * <li>onTopologyChange is triggered and sets scale up timer to 15 and scale down to 110;</li>
+ * <li>for some reason scheduled executor hangs and doesn't trigger timers (let's consider case when everything is recalculated in
+ * onTopologyChange);</li>
+ * <li>node B leaves at time 20;</li>
+ * <li>onTopologyChange is triggered again and sees that scale up timer should have been already triggered (its time to trigger
+ * was 15), it adds node C automatically to new history entry. Scale down timer is scheduled and has to wait more, but
+ * another node (B) left, so scale down timer's nodes are now A and B, and its time to trigger is rescheduled and is now
+ * 20 + 100 = 120. Data nodes in the new data nodes history entry are A, B, C.</li>
+ * </ul>
+ *
+ * @param zoneDescriptor Zone descriptor.
+ * @param revision Meta storage revision.
+ * @param timestamp Timestamp, that is consistent with meta storage revision.
+ * @param newLogicalTopology New logical topology.
+ * @param oldLogicalTopology Old logical topology.
+ * @return CompletableFuture that is completed when the operation is done.
+ */
+ CompletableFuture<Void> onTopologyChange(
+ CatalogZoneDescriptor zoneDescriptor,
+ long revision,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> newLogicalTopology,
+ Set<NodeWithAttributes> oldLogicalTopology
+ ) {
+ int zoneId = zoneDescriptor.id();
+
+ return doOperation(
+ zoneDescriptor,
+ List.of(zoneDataNodesHistoryKey(zoneId), zoneScaleUpTimerKey(zoneId), zoneScaleDownTimerKey(zoneId)),
+ dataNodesHistoryContext -> completedFuture(onTopologyChangeInternal(
+ zoneDescriptor,
+ revision,
+ timestamp,
+ newLogicalTopology,
+ oldLogicalTopology,
+ dataNodesHistoryContext
+ ))
+ );
+ }
+
+ @Nullable
+ private DataNodesHistoryMetaStorageOperation onTopologyChangeInternal(
+ CatalogZoneDescriptor zoneDescriptor,
+ long revision,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> newLogicalTopology,
+ Set<NodeWithAttributes> oldLogicalTopology,
+ @Nullable DistributionZonesUtil.DataNodesHistoryContext dataNodesHistoryContext
+ ) {
+ if (dataNodesHistoryContext == null) {
+ // This means that the zone was not initialized yet. The initial history entry with current topology will
+ // be written on the zone init.
+ return null;
+ }
+
+ DataNodesHistory dataNodesHistory = dataNodesHistoryContext.dataNodesHistory();
+
+ if (dataNodesHistory.entryIsPresentAtExactTimestamp(timestamp)) {
+ // This event was already processed by another node.
+ return null;
+ }
+
+ int zoneId = zoneDescriptor.id();
+
+ LOG.debug("Topology change detected [zoneId={}, timestamp={}, newTopology={}, oldTopology={}].", zoneId, timestamp,
+ nodeNames(newLogicalTopology), nodeNames(oldLogicalTopology));
+
+ DistributionZoneTimer scaleUpTimer = dataNodesHistoryContext.scaleUpTimer();
+ DistributionZoneTimer scaleDownTimer = dataNodesHistoryContext.scaleDownTimer();
+
+ DataNodesHistoryEntry latestDataNodes = dataNodesHistory.dataNodesForTimestamp(timestamp);
+
+ Set<NodeWithAttributes> addedNodes = newLogicalTopology.stream()
+ .filter(node -> !latestDataNodes.dataNodes().contains(node))
+ .collect(toSet());
+
+ Set<NodeWithAttributes> addedNodesComparingToOldTopology = newLogicalTopology.stream()
+ .filter(node -> !oldLogicalTopology.contains(node))
+ .collect(toSet());
+
+ Set<NodeWithAttributes> removedNodes = latestDataNodes.dataNodes().stream()
+ .filter(node -> !newLogicalTopology.contains(node) && !Objects.equals(node.nodeName(), localNodeName))
+ .filter(node -> !scaleDownTimer.nodes().contains(node))
+ .collect(toSet());
+
+ if ((!addedNodes.isEmpty() || !removedNodes.isEmpty())
+ && zoneDescriptor.dataNodesAutoAdjust() != INFINITE_TIMER_VALUE) {
+ // TODO: IGNITE-18134 Create scheduler with dataNodesAutoAdjust timer.
+ throw new UnsupportedOperationException("Data nodes auto adjust is not supported.");
+ }
+
+ int partitionResetDelay = partitionDistributionResetTimeoutSupplier.getAsInt();
+
+ if (!removedNodes.isEmpty()
+ && zoneDescriptor.consistencyMode() == HIGH_AVAILABILITY
+ && partitionResetDelay != INFINITE_TIMER_VALUE) {
+ reschedulePartitionReset(partitionResetDelay, () -> partitionResetClosure.accept(revision, zoneId), zoneId);
+ }
+
+ DistributionZoneTimer mergedScaleUpTimer = mergeTimerOnTopologyChange(zoneDescriptor, timestamp,
+ scaleUpTimer, addedNodes, newLogicalTopology, true);
+
+ DistributionZoneTimer mergedScaleDownTimer = mergeTimerOnTopologyChange(zoneDescriptor, timestamp,
+ scaleDownTimer, removedNodes, newLogicalTopology, false);
+
+ DataNodesHistoryEntry currentDataNodes = currentDataNodes(
+ timestamp,
+ dataNodesHistory,
+ mergedScaleUpTimer,
+ mergedScaleDownTimer,
+ zoneDescriptor
+ );
+
+ DistributionZoneTimer scaleUpTimerToSave = timerToSave(timestamp, mergedScaleUpTimer);
+ DistributionZoneTimer scaleDownTimerToSave = timerToSave(timestamp, mergedScaleDownTimer);
+
+ boolean addMandatoryEntry = !addedNodesComparingToOldTopology.isEmpty();
+
+ Condition condition = and(
+ dataNodesHistoryEqualToOrNotExists(zoneId, dataNodesHistory),
+ and(
+ timerEqualToOrNotExists(zoneScaleUpTimerKey(zoneId), scaleUpTimer),
+ timerEqualToOrNotExists(zoneScaleDownTimerKey(zoneId), scaleDownTimer)
+ )
+ );
+
+ List<Operation> operations = List.of(
+ addNewEntryToDataNodesHistory(zoneId, dataNodesHistory, currentDataNodes.timestamp(),
+ currentDataNodes.dataNodes(), addMandatoryEntry),
+ renewTimer(zoneScaleUpTimerKey(zoneId), scaleUpTimerToSave),
+ renewTimer(zoneScaleDownTimerKey(zoneId), scaleDownTimerToSave)
+ );
+
+ return DataNodesHistoryMetaStorageOperation.builder()
+ .zoneId(zoneId)
+ .condition(condition)
+ .operations(operations)
+ .operationName("topology change")
+ .currentDataNodesHistory(dataNodesHistory)
+ .currentTimestamp(timestamp)
+ .historyEntryTimestamp(currentDataNodes.timestamp())
+ .historyEntryNodes(currentDataNodes.dataNodes())
+ .scaleUpTimer(scaleUpTimerToSave)
+ .scaleDownTimer(scaleDownTimerToSave)
+ .addMandatoryEntry(addMandatoryEntry)
+ .build();
+ }
+
+ /**
+ * This is called on topology change, when some nodes may be added or removed, and therefore scale up or scale down timers can be
+ * created. By this moment there can be already some timers in meta storage, so we need to merge them with new topology changes, that
+ * is done by this method. The create time of the new timer is set to current timestamp, and sets of nodes in two timers are unified,
+ * with consideration of the logical topology: if the node is absent in topology, it is excluded from scale up timer, and if it is
+ * present, it is excluded from scale down timer.
+ *
+ * @param zoneDescriptor Zone descriptor.
+ * @param timestamp Current timestamp.
+ * @param currentTimer Current timer, that is taken from meta storage.
+ * @param nodes Set of nodes that are added or removed, depending on the timer, this is defined by {@code scaleUp} parameter.
+ * @param logicalTopology Current logical topology.
+ * @param scaleUp If true, the timer is scale up, otherwise it is scale down.
+ * @return Merged timer.
+ */
+ private static DistributionZoneTimer mergeTimerOnTopologyChange(
+ CatalogZoneDescriptor zoneDescriptor,
+ HybridTimestamp timestamp,
+ DistributionZoneTimer currentTimer,
+ Set<NodeWithAttributes> nodes,
+ Set<NodeWithAttributes> logicalTopology,
+ boolean scaleUp
+ ) {
+ // Filter the current timer's nodes according to the current topology, if it is newer than the timer's timestamp.
+ Set<NodeWithAttributes> currentTimerFilteredNodes = currentTimer.nodes().stream()
+ .filter(n -> {
+ if (currentTimer.createTimestamp().longValue() >= timestamp.longValue()) {
+ return true;
+ } else {
+ return scaleUp == nodeNames(logicalTopology).contains(n.nodeName());
+ }
+ })
+ .collect(toSet());
+
+ if (nodes.isEmpty()) {
+ return new DistributionZoneTimer(
+ currentTimer.createTimestamp(),
+ currentTimer.timeToWaitInSeconds(),
+ currentTimerFilteredNodes
+ );
+ } else {
+ int autoAdjustWaitInSeconds = scaleUp
+ ? zoneDescriptor.dataNodesAutoAdjustScaleUp()
+ : zoneDescriptor.dataNodesAutoAdjustScaleDown();
+
+ return new DistributionZoneTimer(
+ timestamp,
+ autoAdjustWaitInSeconds,
+ union(nodes, currentTimerFilteredNodes)
+ );
+ }
+ }
+
+ /**
+ * Returns timer value to save in the meta storage. If the timer is already applied according to current timestamp,
+ * returns default timer.
+ *
+ * @param currentTimestamp Current timestamp.
+ * @param timer Timer.
+ * @return Timer to save.
+ */
+ private static DistributionZoneTimer timerToSave(HybridTimestamp currentTimestamp, DistributionZoneTimer timer) {
+ return timer.timeToTrigger().longValue() <= currentTimestamp.longValue() ? DEFAULT_TIMER : timer;
+ }
+
+ /**
+ * Recalculates data nodes on zone filter changes according only to current topology and new filters. Stops and discards all timers.
+ *
+ * @param zoneDescriptor Zone descriptor.
+ * @param timestamp Current timestamp.
+ * @param logicalTopology New logical topology.
+ * @return CompletableFuture that is completed when the operation is done.
+ */
+ CompletableFuture<Void> onZoneFilterChange(
+ CatalogZoneDescriptor zoneDescriptor,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> logicalTopology
+ ) {
+ int zoneId = zoneDescriptor.id();
+
+ return doOperation(
+ zoneDescriptor,
+ List.of(zoneDataNodesHistoryKey(zoneId)),
+ dataNodesHistoryContext -> completedFuture(onZoneFilterChangeInternal(
+ zoneDescriptor,
+ timestamp,
+ logicalTopology,
+ dataNodesHistoryContext
+ ))
+ );
+ }
+
+ @Nullable
+ private DataNodesHistoryMetaStorageOperation onZoneFilterChangeInternal(
+ CatalogZoneDescriptor zoneDescriptor,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> logicalTopology,
+ DataNodesHistoryContext dataNodesHistoryContext
+ ) {
+ assert dataNodesHistoryContext != null : "Data nodes history and timers are missing, zone=" + zoneDescriptor;
+
+ DataNodesHistory dataNodesHistory = dataNodesHistoryContext.dataNodesHistory();
+
+ if (dataNodesHistory.entryIsPresentAtExactTimestamp(timestamp)) {
+ return null;
+ }
+
+ int zoneId = zoneDescriptor.id();
+
+ LOG.debug("Distribution zone filter changed [zoneId={}, timestamp={}, logicalTopology={}, descriptor={}].", zoneId,
+ timestamp, nodeNames(logicalTopology), zoneDescriptor);
+
+ stopAllTimers(zoneId);
+
+ Set<NodeWithAttributes> dataNodes = filterDataNodes(logicalTopology, zoneDescriptor);
+
+ return DataNodesHistoryMetaStorageOperation.builder()
+ .zoneId(zoneId)
+ .condition(dataNodesHistoryEqualToOrNotExists(zoneId, dataNodesHistory))
+ .operations(List.of(
+ addNewEntryToDataNodesHistory(zoneId, dataNodesHistory, timestamp, dataNodes),
+ clearTimer(zoneScaleUpTimerKey(zoneId)),
+ clearTimer(zoneScaleDownTimerKey(zoneId)),
+ clearTimer(zonePartitionResetTimerKey(zoneId))
+ ))
+ .operationName("distribution zone filter change")
+ .currentDataNodesHistory(dataNodesHistory)
+ .currentTimestamp(timestamp)
+ .historyEntryTimestamp(timestamp)
+ .historyEntryNodes(dataNodes)
+ .scaleUpTimer(DEFAULT_TIMER)
+ .scaleDownTimer(DEFAULT_TIMER)
+ .build();
+ }
+
+ /**
+ * Recalculates data nodes on zone auto adjust alterations. Modifies scale up and scale down timers and appends data nodes history.
+ * The new data nodes are calculated according to the new timer values. See also description of {@link #onTopologyChange} method
+ * for more details and examples of how data nodes are recalculated.
+ *
+ * @param zoneDescriptor Zone descriptor.
+ * @param timestamp Current timestamp.
+ * @param oldAutoAdjustScaleUp Old scale up auto adjust value.
+ * @param oldAutoAdjustScaleDown Old scale down auto adjust value.
+ * @return CompletableFuture that is completed when the operation is done.
+ */
+ CompletableFuture<Void> onAutoAdjustAlteration(
+ CatalogZoneDescriptor zoneDescriptor,
+ HybridTimestamp timestamp,
+ int oldAutoAdjustScaleUp,
+ int oldAutoAdjustScaleDown
+ ) {
+ int zoneId = zoneDescriptor.id();
+
+ return doOperation(
+ zoneDescriptor,
+ List.of(zoneDataNodesHistoryKey(zoneId), zoneScaleUpTimerKey(zoneId), zoneScaleDownTimerKey(zoneId)),
+ dataNodesHistoryContext -> completedFuture(onAutoAdjustAlterationInternal(
+ zoneDescriptor,
+ timestamp,
+ oldAutoAdjustScaleUp,
+ oldAutoAdjustScaleDown,
+ dataNodesHistoryContext
+ ))
+ );
+ }
+
+ @Nullable
+ private DataNodesHistoryMetaStorageOperation onAutoAdjustAlterationInternal(
+ CatalogZoneDescriptor zoneDescriptor,
+ HybridTimestamp timestamp,
+ int oldAutoAdjustScaleUp,
+ int oldAutoAdjustScaleDown,
+ DataNodesHistoryContext dataNodesHistoryContext
+ ) {
+ assert dataNodesHistoryContext != null : "Data nodes history and timers are missing, zone=" + zoneDescriptor;
+
+ DataNodesHistory dataNodesHistory = dataNodesHistoryContext.dataNodesHistory();
+
+ if (dataNodesHistory.entryIsPresentAtExactTimestamp(timestamp)) {
+ return null;
+ }
+
+ int zoneId = zoneDescriptor.id();
+
+ LOG.debug("Distribution zone auto adjust changed [zoneId={}, timestamp={}, oldAutoAdjustScaleUp={}, "
+ + ", oldAutoAdjustScaleDown={}, descriptor={}].",
+ zoneId, timestamp, oldAutoAdjustScaleUp, oldAutoAdjustScaleDown, zoneDescriptor);
+
+ DistributionZoneTimer scaleUpTimer = dataNodesHistoryContext.scaleUpTimer();
+ DistributionZoneTimer scaleDownTimer = dataNodesHistoryContext.scaleDownTimer();
+
+ DistributionZoneTimer modifiedScaleUpTimer = scaleUpTimer
+ .modifyTimeToWait(zoneDescriptor.dataNodesAutoAdjustScaleUp());
+
+ DistributionZoneTimer modifiedScaleDownTimer = scaleDownTimer
+ .modifyTimeToWait(zoneDescriptor.dataNodesAutoAdjustScaleDown());
+
+ DataNodesHistoryEntry currentDataNodes = currentDataNodes(
+ timestamp,
+ dataNodesHistory,
+ modifiedScaleUpTimer,
+ modifiedScaleDownTimer,
+ zoneDescriptor
+ );
+
+ DistributionZoneTimer scaleUpTimerToSave = timerToSave(timestamp, modifiedScaleUpTimer);
+ DistributionZoneTimer scaleDownTimerToSave = timerToSave(timestamp, modifiedScaleDownTimer);
+
+ Condition condition = and(
+ dataNodesHistoryEqualToOrNotExists(zoneId, dataNodesHistory),
+ and(
+ timerEqualToOrNotExists(zoneScaleUpTimerKey(zoneId), scaleUpTimer),
+ timerEqualToOrNotExists(zoneScaleDownTimerKey(zoneId), scaleDownTimer)
+ )
+ );
+
+ List<Operation> operations = List.of(
+ addNewEntryToDataNodesHistory(zoneId, dataNodesHistory, currentDataNodes.timestamp(),
+ currentDataNodes.dataNodes()),
+ renewTimer(zoneScaleUpTimerKey(zoneId), scaleUpTimerToSave),
+ renewTimer(zoneScaleDownTimerKey(zoneId), scaleDownTimerToSave)
+ );
+
+ return DataNodesHistoryMetaStorageOperation.builder()
+ .zoneId(zoneId)
+ .condition(condition)
+ .operations(operations)
+ .operationName("distribution zone auto adjust change")
+ .currentDataNodesHistory(dataNodesHistory)
+ .currentTimestamp(timestamp)
+ .historyEntryTimestamp(currentDataNodes.timestamp())
+ .historyEntryNodes(currentDataNodes.dataNodes())
+ .scaleUpTimer(scaleUpTimerToSave)
+ .scaleDownTimer(scaleDownTimerToSave)
+ .build();
+ }
+
+ void onUpdatePartitionDistributionReset(
+ int zoneId,
+ int partitionDistributionResetTimeoutSeconds,
+ Runnable taskOnReset
+ ) {
+ if (partitionDistributionResetTimeoutSeconds == INFINITE_TIMER_VALUE) {
+ zoneTimers.computeIfAbsent(zoneId, this::createZoneTimers).partitionReset.stopScheduledTask();
+ } else {
+ zoneTimers.computeIfAbsent(zoneId, this::createZoneTimers).partitionReset
+ .reschedule(partitionDistributionResetTimeoutSeconds, taskOnReset);
+ }
+ }
+
+ /**
+ * The closure that is executed on scale up or scale down schedule. Appends data nodes history in meta storage and sets the timer
+ * to default value.
+ *
+ * @param zoneDescriptor Zone descriptor.
+ * @param scheduledTimer Scheduled timer.
+ * @return Runnable that is executed on schedule.
+ */
+ private Runnable applyTimerClosure(CatalogZoneDescriptor zoneDescriptor, ScheduledTimer scheduledTimer) {
+ int zoneId = zoneDescriptor.id();
+
+ return () -> doOperation(
+ zoneDescriptor,
+ List.of(zoneDataNodesHistoryKey(zoneId), scheduledTimer.metaStorageKey()),
+ dataNodesHistoryContext -> applyTimerClosure0(zoneDescriptor, scheduledTimer, dataNodesHistoryContext)
+ );
+ }
+
+ @Nullable
+ private CompletableFuture<DataNodesHistoryMetaStorageOperation> applyTimerClosure0(
+ CatalogZoneDescriptor zoneDescriptor,
+ ScheduledTimer scheduledTimer,
+ DataNodesHistoryContext dataNodesHistoryContext
+ ) {
+ assert dataNodesHistoryContext != null : "Data nodes history and timers are missing, zone=" + zoneDescriptor;
+
+ DataNodesHistory dataNodesHistory = dataNodesHistoryContext.dataNodesHistory();
+
+ DistributionZoneTimer timer = scheduledTimer.timerFromContext(dataNodesHistoryContext);
+
+ if (timer.equals(DEFAULT_TIMER)) {
+ return nullCompletedFuture();
+ }
+
+ int zoneId = zoneDescriptor.id();
+
+ LOG.debug("Triggered " + scheduledTimer.name() + " [zoneId={}, timer={}].", zoneId, timer);
+
+ HybridTimestamp timeToTrigger = timer.timeToTrigger();
+
+ if (dataNodesHistory.entryIsPresentAtExactTimestamp(timeToTrigger)) {
+ return nullCompletedFuture();
+ }
+
+ long currentDelay = delayInSeconds(timeToTrigger);
+
+ if (currentDelay < 0) {
+ return nullCompletedFuture();
+ }
+
+ if (currentDelay > 1) {
+ scheduledTimer.reschedule(currentDelay, applyTimerClosure(zoneDescriptor, scheduledTimer));
+
+ return nullCompletedFuture();
+ }
+
+ DataNodesHistoryEntry currentDataNodes = scheduledTimer
+ .recalculateDataNodes(dataNodesHistory, timer);
+
+ // We need to wait for actual time according to hybrid clock plus clock skew, because scheduled executor
+ // is not synchronized with hybrid clock. If we don't do this and scheduled executor will trigger this closure
+ // earlier than max hybrid clock time in cluster, we may append new history entry having timestamp that
+ // is (actually) in future.
+ return clockService.waitFor(timeToTrigger.addPhysicalTime(clockService.maxClockSkewMillis()))
+ .thenApply(ignored -> DataNodesHistoryMetaStorageOperation.builder()
+ .zoneId(zoneId)
+ .condition(
+ and(
+ dataNodesHistoryEqualToOrNotExists(zoneId, dataNodesHistory),
+ timerEqualToOrNotExists(scheduledTimer.metaStorageKey(), timer)
+ )
+ )
+ .operations(
+ List.of(
+ addNewEntryToDataNodesHistory(
+ zoneId,
+ dataNodesHistory,
+ timeToTrigger,
+ currentDataNodes.dataNodes()
+ ),
+ clearTimer(scheduledTimer.metaStorageKey())
+ )
+ )
+ .operationName(scheduledTimer.name() + " trigger")
+ .currentDataNodesHistory(dataNodesHistory)
+ .currentTimestamp(timeToTrigger)
+ .historyEntryTimestamp(timeToTrigger)
+ .historyEntryNodes(currentDataNodes.dataNodes())
+ .scaleUpTimer(scheduledTimer.scaleUpTimerAfterApply())
+ .scaleDownTimer(scheduledTimer.scaleDownTimerAfterApply())
+ .build()
+ );
+ }
+
+ private void onScaleUpTimerChange(CatalogZoneDescriptor zoneDescriptor, DistributionZoneTimer scaleUpTimer) {
+ ScheduledTimer timer = new ScaleUpScheduledTimer(zoneDescriptor);
+ timer.init(scaleUpTimer);
+ }
+
+ private void onScaleDownTimerChange(CatalogZoneDescriptor zoneDescriptor, DistributionZoneTimer scaleDownTimer) {
+ ScheduledTimer timer = new ScaleDownScheduledTimer(zoneDescriptor);
+ timer.init(scaleDownTimer);
+ }
+
+ private void restorePartitionResetTimer(int zoneId, DistributionZoneTimer scaleDownTimer, long revision) {
+ if (!scaleDownTimer.equals(DEFAULT_TIMER)) {
+ reschedulePartitionReset(
+ partitionDistributionResetTimeoutSupplier.getAsInt(),
+ () -> partitionResetClosure.accept(revision, zoneId),
+ zoneId
+ );
+ }
+ }
+
+ private static long delayInSeconds(HybridTimestamp timerTimestamp) {
+ if (timerTimestamp.equals(HybridTimestamp.MIN_VALUE) || timerTimestamp.equals(HybridTimestamp.MAX_VALUE)) {
+ return -1;
+ }
+
+ long currentTime = System.currentTimeMillis();
+
+ long delayMs = timerTimestamp.getPhysical() - currentTime;
+
+ return max(0, delayMs / 1000);
+ }
+
+ private void reschedulePartitionReset(long delayInSeconds, Runnable runnable, int zoneId) {
+ zoneTimers.computeIfAbsent(zoneId, this::createZoneTimers).partitionReset.reschedule(delayInSeconds, runnable);
+ }
+
+ private ZoneTimers createZoneTimers(int zoneId) {
+ return new ZoneTimers(zoneId, executor);
+ }
+
+ /**
+ * Calculates current data nodes on topology changes and distribution zone changes, according to the timers' state.
+ * This method is NOT used for data nodes calculation for external components, for that see {@link #dataNodes(int, HybridTimestamp)}.
+ *
+ * @param timestamp Current timestamp.
+ * @param dataNodesHistory Data nodes history.
+ * @param scaleUpTimer Scale up timer.
+ * @param scaleDownTimer Scale down timer.
+ * @param zoneDescriptor Zone descriptor.
+ * @return History entry of current timestamp and data nodes.
+ */
+ private static DataNodesHistoryEntry currentDataNodes(
+ HybridTimestamp timestamp,
+ DataNodesHistory dataNodesHistory,
+ DistributionZoneTimer scaleUpTimer,
+ DistributionZoneTimer scaleDownTimer,
+ CatalogZoneDescriptor zoneDescriptor
+ ) {
+ DataNodesHistoryEntry currentDataNodesEntry = dataNodesHistory.dataNodesForTimestamp(timestamp);
+
+ assert currentDataNodesEntry.timestamp().longValue() != HybridTimestamp.MIN_VALUE.longValue()
+ : "Data nodes history is missing for timestamp [zoneId=" + zoneDescriptor.id() + ", timestamp=" + timestamp + "].";
+
+ Set<NodeWithAttributes> dataNodes = new HashSet<>(currentDataNodesEntry.dataNodes());
+
+ long scaleUpTriggerTime = scaleUpTimer.timeToTrigger().longValue();
+ long scaleDownTriggerTime = scaleDownTimer.timeToTrigger().longValue();
+ long timestampLong = timestamp.longValue();
+ HybridTimestamp newTimestamp = timestamp;
+
+ if (scaleUpTriggerTime <= timestampLong) {
+ newTimestamp = scaleUpTimer.timeToTrigger();
+ dataNodes.addAll(filterDataNodes(scaleUpTimer.nodes(), zoneDescriptor));
+ }
+
+ if (scaleDownTriggerTime <= timestampLong) {
+ dataNodes.removeAll(scaleDownTimer.nodes());
+
+ if (scaleDownTriggerTime > scaleUpTriggerTime) {
+ newTimestamp = scaleDownTimer.timeToTrigger();
+ }
+ }
+
+ return new DataNodesHistoryEntry(newTimestamp, dataNodes);
+ }
+
+ /**
+ * Returns data nodes for the given zone and timestamp. See {@link #dataNodes(int, HybridTimestamp)}.
+ * Catalog version is calculated by the given timestamp.
+ *
+ * @param zoneId Zone ID.
+ * @param timestamp Timestamp.
+ * @return Data nodes.
+ */
+ public CompletableFuture<Set<String>> dataNodes(int zoneId, HybridTimestamp timestamp) {
+ return dataNodes(zoneId, timestamp, null);
+ }
+
+ /**
+ * Returns data nodes for the given zone and timestamp. It doesn't recalculate the data nodes, it just retrieves them from data nodes
+ * history. For information how data nodes are calculated, see {@link #onTopologyChange}, {@link #onZoneFilterChange} and {@link
+ * #onAutoAdjustAlteration} methods.
+ *
+ * @param zoneId Zone ID.
+ * @param timestamp Timestamp.
+ * @param catalogVersion Catalog version.
+ * @return Data nodes.
+ */
+ public CompletableFuture<Set<String>> dataNodes(int zoneId, HybridTimestamp timestamp, @Nullable Integer catalogVersion) {
+ DataNodesHistory volatileHistory = dataNodesHistoryVolatile.get(zoneId);
+ if (volatileHistory != null && volatileHistory.entryIsPresentAtExactTimestamp(timestamp)) {
+ return completedFuture(volatileHistory.dataNodesForTimestamp(timestamp)
+ .dataNodes()
+ .stream()
+ .map(NodeWithAttributes::nodeName)
+ .collect(toSet())
+ );
+ }
+
+ if (catalogVersion == null) {
+ catalogVersion = catalogManager.activeCatalogVersion(timestamp.longValue());
+ }
+
+ CatalogZoneDescriptor zone = catalogManager.catalog(catalogVersion).zone(zoneId);
+
+ if (zone == null) {
+ return failedFuture(new DistributionZoneNotFoundException(zoneId));
+ }
+
+ return getValueFromMetaStorage(zoneDataNodesHistoryKey(zoneId), DataNodesHistorySerializer::deserialize)
+ .thenApply(history -> inBusyLock(busyLock, () -> {
+ if (history == null) {
+ // It means that the zone was created but the data nodes value had not been updated yet.
+ // So the data nodes value will be equals to the logical topology.
+ return filterDataNodes(topologyNodes(), zone);
+ }
+
+ DataNodesHistoryEntry entry = history.dataNodesForTimestamp(timestamp);
+
+ return entry.dataNodes();
+ }))
+ .thenApply(nodes -> nodes.stream().map(NodeWithAttributes::nodeName).collect(toSet()));
+ }
+
+ private Set<NodeWithAttributes> topologyNodes() {
+ // It means that the zone was created but the data nodes value had not been updated yet.
+ // So the data nodes value will be equals to the logical topology on the descLastUpdateRevision.
+ Entry topologyEntry = metaStorageManager.getLocally(zonesLogicalTopologyKey());
+
+ if (topologyEntry.empty()) {
+ // A special case for a very first start of a node, when a creation of a zone was before the first topology event,
+ // meaning that zonesLogicalTopologyKey() has not been updated yet, safe to return empty set here.
+ return emptySet();
+ }
+
+ return deserializeLogicalTopologySet(topologyEntry.value());
+ }
+
+ private static Condition dataNodesHistoryEqualToOrNotExists(int zoneId, DataNodesHistory history) {
+ return or(
+ notExists(zoneDataNodesHistoryKey(zoneId)),
+ value(zoneDataNodesHistoryKey(zoneId)).eq(DataNodesHistorySerializer.serialize(history))
+ );
+ }
+
+ private static Condition timerEqualToOrNotExists(ByteArray timerKey, DistributionZoneTimer timer) {
+ return or(
+ notExists(timerKey),
+ or(
+ value(timerKey).eq(DistributionZoneTimerSerializer.serialize(timer)),
+ value(timerKey).eq(DistributionZoneTimerSerializer.serialize(DEFAULT_TIMER))
+ )
+ );
+ }
+
+ private Operation addNewEntryToDataNodesHistory(
+ int zoneId,
+ DataNodesHistory history,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> nodes
+ ) {
+ return addNewEntryToDataNodesHistory(zoneId, history, timestamp, nodes, false);
+ }
+
+ /**
+ * Meta storage operation that adds new entry to data nodes history.
+ * If the new entry is the same as the latest one {@code addMandatoryEntry} is {@code false}, then no new entry is added.
+ *
+ * @param zoneId Zone id.
+ * @param history Current data nodes history.
+ * @param timestamp Timestamp of the new entry.
+ * @param nodes Data nodes for the new entry.
+ * @param addMandatoryEntry If {@code true}, then the new entry is added even if it is the same as the latest one.
+ * @return Operation that adds new entry to data nodes history.
+ */
+ private Operation addNewEntryToDataNodesHistory(
+ int zoneId,
+ DataNodesHistory history,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> nodes,
+ boolean addMandatoryEntry
+ ) {
+ if (!addMandatoryEntry
+ && !history.isEmpty()
+ && nodes.equals(history.dataNodesForTimestamp(HybridTimestamp.MAX_VALUE).dataNodes())) {
+ return noop();
+ } else {
+ DataNodesHistory newHistory = history.addHistoryEntry(timestamp, nodes);
+ dataNodesHistoryVolatile.put(zoneId, newHistory);
+ return put(zoneDataNodesHistoryKey(zoneId), DataNodesHistorySerializer.serialize(newHistory));
+ }
+ }
+
+ private static Operation renewTimer(ByteArray timerKey, DistributionZoneTimer timer) {
+ return put(timerKey, DistributionZoneTimerSerializer.serialize(timer));
+ }
+
+ private static Operation clearTimer(ByteArray timerKey) {
+ return put(timerKey, DistributionZoneTimerSerializer.serialize(DEFAULT_TIMER));
+ }
+
+ private <T> CompletableFuture<T> getValueFromMetaStorage(ByteArray key, Function<byte[], T> deserializer) {
+ return metaStorageManager.get(key).thenApply(e -> deserializeEntry(e, deserializer));
+ }
+
+ private CompletableFuture<DataNodesHistoryContext> getDataNodeHistoryContextMs(List<ByteArray> keys) {
+ return metaStorageManager.getAll(new HashSet<>(keys))
+ .thenApply(entries -> dataNodeHistoryContextFromValues(entries.values()));
+ }
+
+ private CompletableFuture<DataNodesHistoryContext> getDataNodeHistoryContextMsLocally(List<ByteArray> keys) {
+ List<Entry> entries = metaStorageManager.getAllLocally(keys);
+
+ return completedFuture(dataNodeHistoryContextFromValues(entries));
+ }
+
+ @Nullable
+ private static <T> T deserializeEntry(@Nullable Entry e, Function<byte[], T> deserializer) {
+ if (e == null || e.value() == null || e.empty() || e.tombstone()) {
+ return null;
+ } else {
+ return deserializer.apply(e.value());
+ }
+ }
+
+ /**
+ * Perform a meta storage operation for changing data nodes history and timers. Uses {@link #msInvokeWithRetry} for retries.
+ *
+ * @param zone Zone descriptor.
+ * @param keysToRead Keys to read from meta storage, to get {@link DataNodesHistoryContext}.
+ * @param operation Operation.
+ * @return Future reflecting the completion of meta storage operation.
+ */
+ private CompletableFuture<Void> doOperation(
+ CatalogZoneDescriptor zone,
+ List<ByteArray> keysToRead,
+ Function<DataNodesHistoryContext, CompletableFuture<DataNodesHistoryMetaStorageOperation>> operation
+ ) {
+ return msInvokeWithRetry(msGetter -> msGetter.get(keysToRead).thenCompose(operation), zone);
+ }
+
+ private CompletableFuture<Void> msInvokeWithRetry(
+ Function<
+ DataNodeHistoryContextMetaStorageGetter,
+ CompletableFuture<DataNodesHistoryMetaStorageOperation>
+ > metaStorageOperationSupplier,
+ CatalogZoneDescriptor zone
+ ) {
+ return msInvokeWithRetry(metaStorageOperationSupplier, MAX_ATTEMPTS_ON_RETRY, zone);
+ }
+
+ /**
+ * Utility method for meta storage invocation.
+ *
+ * @param metaStorageOperationSupplier Function that returns IIF operation to use in meta storage invocation.
+ * It contains the actual logic of operation.
+ * @param attemptsLeft Number of attempts left, used if meta storage invoke did not succeed due to CAS fail.
+ * @param zone Zone descriptor.
+ * @return Future reflecting the completion of meta storage invocation.
+ */
+ private CompletableFuture<Void> msInvokeWithRetry(
+ Function<
+ DataNodeHistoryContextMetaStorageGetter,
+ CompletableFuture<DataNodesHistoryMetaStorageOperation>
+ > metaStorageOperationSupplier,
+ int attemptsLeft,
+ CatalogZoneDescriptor zone
+ ) {
+ if (attemptsLeft <= 0) {
+ throw new AssertionError("Failed to perform meta storage invoke, maximum number of attempts reached [zone=" + zone + "].");
+ }
+
+ // Get locally on the first attempt, otherwise it means that invoke has failed because of different value in meta storage,
+ // so we need to retrieve the value from meta storage.
+ DataNodeHistoryContextMetaStorageGetter msGetter = attemptsLeft == MAX_ATTEMPTS_ON_RETRY
+ ? this::getDataNodeHistoryContextMsLocally
+ : this::getDataNodeHistoryContextMs;
+
+ CompletableFuture<DataNodesHistoryMetaStorageOperation> metaStorageOperationFuture =
+ metaStorageOperationSupplier.apply(msGetter);
+
+ return metaStorageOperationFuture
+ .thenCompose(metaStorageOperation -> {
+ if (metaStorageOperation == null) {
+ return nullCompletedFuture();
+ } else {
+ // TODO https://issues.apache.org/jira/browse/IGNITE-24611
+ return metaStorageManager.invoke(metaStorageOperation.operation())
+ .thenCompose(result -> {
+ if (result.getAsBoolean()) {
+ LOG.info(metaStorageOperation.successLogMessage());
+
+ return nullCompletedFuture();
+ } else {
+ return msInvokeWithRetry(metaStorageOperationSupplier, attemptsLeft - 1, zone);
+ }
+ })
+ .whenComplete((v, e) -> {
+ if (e != null) {
+ LOG.error(metaStorageOperation.failureLogMessage(), e);
+ }
+ });
+ }
+ });
+ }
+
+ /**
+ * Method initialise data nodes history value for the specified zone, also sets the
+ * {@link DistributionZonesUtil#zoneScaleUpTimerKey(int)} and {@link DistributionZonesUtil#zoneScaleDownTimerKey}
+ * if it passes the condition. It is called on the first creation of a zone.
+ *
+ * @param zoneId Unique id of a zone
+ * @param timestamp Timestamp of an event that has triggered this method.
+ * @param dataNodes Data nodes.
+ * @return Future reflecting the completion of initialisation of zone's keys in meta storage.
+ */
+ CompletableFuture<?> onZoneCreate(
+ int zoneId,
+ HybridTimestamp timestamp,
+ Set<NodeWithAttributes> dataNodes
+ ) {
+ if (!busyLock.enterBusy()) {
+ throw new IgniteInternalException(NODE_STOPPING_ERR, new NodeStoppingException());
+ }
+
+ try {
+ // Update data nodes for a zone only if the corresponding data nodes keys weren't initialised in ms yet.
+ Condition condition = and(
+ notExists(zoneDataNodesHistoryKey(zoneId)),
+ notTombstone(zoneDataNodesHistoryKey(zoneId))
+ );
+
+ Update update = ops(
+ addNewEntryToDataNodesHistory(zoneId, new DataNodesHistory(), timestamp, dataNodes),
+ clearTimer(zoneScaleUpTimerKey(zoneId)),
+ clearTimer(zoneScaleDownTimerKey(zoneId)),
+ clearTimer(zonePartitionResetTimerKey(zoneId))
+ ).yield(true);
+
+ Iif iif = iif(condition, update, ops().yield(false));
+
+ return metaStorageManager.invoke(iif)
+ .thenApply(StatementResult::getAsBoolean)
+ .whenComplete((invokeResult, e) -> {
+ if (e != null) {
+ LOG.error(
+ "Failed to initialize zone's dataNodes history [zoneId = {}, timestamp = {}, dataNodes = {}]",
+ e,
+ zoneId,
+ timestamp,
+ nodeNames(dataNodes)
+ );
+ } else if (invokeResult) {
+ LOG.info("Initialized zone's dataNodes history [zoneId = {}, timestamp = {}, dataNodes = {}]",
+ zoneId,
+ timestamp,
+ nodeNames(dataNodes)
+ );
+ } else {
+ LOG.debug(
+ "Failed to initialize zone's dataNodes history [zoneId = {}, timestamp = {}, dataNodes = {}]",
+ zoneId,
+ timestamp,
+ nodeNames(dataNodes)
+ );
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ CompletableFuture<?> onZoneDrop(int zoneId, HybridTimestamp timestamp) {
+ return removeDataNodesKeys(zoneId, timestamp)
+ .thenRun(() -> {
+ ZoneTimers zt = zoneTimers.remove(zoneId);
+ if (zt != null) {
+ zt.stopAllTimers();
+ }
+ });
+ }
+
+ /**
+ * Method deletes data nodes related values for the specified zone.
+ *
+ * @param zoneId Unique id of a zone.
+ * @param timestamp Timestamp of an event that has triggered this method.
+ */
+ private CompletableFuture<?> removeDataNodesKeys(int zoneId, HybridTimestamp timestamp) {
+ if (!busyLock.enterBusy()) {
+ throw new IgniteInternalException(NODE_STOPPING_ERR, new NodeStoppingException());
+ }
+
+ try {
+ Condition condition = exists(zoneDataNodesHistoryKey(zoneId));
+
+ Update removeKeysUpd = ops(
+ // TODO remove(zoneDataNodesHistoryKey(zoneId)), https://issues.apache.org/jira/browse/IGNITE-24345
+ remove(zoneScaleUpTimerKey(zoneId)),
+ remove(zoneScaleDownTimerKey(zoneId)),
+ remove(zonePartitionResetTimerKey(zoneId))
+ ).yield(true);
+
+ Iif iif = iif(condition, removeKeysUpd, ops().yield(false));
+
+ return metaStorageManager.invoke(iif)
+ .thenApply(StatementResult::getAsBoolean)
+ .whenComplete((invokeResult, e) -> {
+ if (e != null) {
+ LOG.error(
+ "Failed to delete zone's dataNodes keys [zoneId = {}, timestamp = {}]",
+ e,
+ zoneId,
+ timestamp
+ );
+ } else if (invokeResult) {
+ LOG.info("Delete zone's dataNodes keys [zoneId = {}, timestamp = {}]", zoneId, timestamp);
+ } else {
+ LOG.debug("Failed to delete zone's dataNodes keys [zoneId = {}, timestamp = {}]", zoneId, timestamp);
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ private void stopAllTimers(int zoneId) {
+ ZoneTimers zt = zoneTimers.get(zoneId);
+ if (zt != null) {
+ zt.stopAllTimers();
+ }
+ }
+
+ private WatchListener createScaleUpTimerPrefixListener() {
+ return event -> inBusyLockAsync(
+ busyLock,
+ () -> processWatchEvent(
+ event,
+ DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX_BYTES,
+ (zoneId, e) -> processTimerWatchEvent(zoneId, e, true)
+ )
+ );
+ }
+
+ private WatchListener createScaleDownTimerPrefixListener() {
+ return event -> inBusyLockAsync(
+ busyLock,
+ () -> processWatchEvent(
+ event,
+ DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX_BYTES,
+ (zoneId, e) -> processTimerWatchEvent(zoneId, e, false)
+ )
+ );
+ }
+
+ private WatchListener createDataNodesListener() {
+ return event -> inBusyLockAsync(
+ busyLock,
+ () -> processWatchEvent(event, DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES, this::processDataNodesHistoryWatchEvent)
+ );
+ }
+
+ private CompletableFuture<Void> processTimerWatchEvent(int zoneId, Entry e, boolean scaleUp) {
+ if (e.tombstone()) {
+ return nullCompletedFuture();
+ }
+
+ DistributionZoneTimer timer = DistributionZoneTimerSerializer.deserialize(e.value());
+
+ CatalogZoneDescriptor zoneDescriptor = catalogManager.activeCatalog(e.timestamp().longValue()).zone(zoneId);
+
+ if (zoneDescriptor == null) {
+ return nullCompletedFuture();
+ }
+
+ if (scaleUp) {
+ onScaleUpTimerChange(zoneDescriptor, timer);
+ } else {
+ onScaleDownTimerChange(zoneDescriptor, timer);
+ }
+
+ return nullCompletedFuture();
+ }
+
+ private CompletableFuture<Void> processDataNodesHistoryWatchEvent(int zoneId, Entry e) {
+ if (!e.tombstone() && e.value() != null) {
+ DataNodesHistory history = DataNodesHistorySerializer.deserialize(e.value());
+
+ CatalogZoneDescriptor zoneDescriptor = catalogManager.activeCatalog(e.timestamp().longValue()).zone(zoneId);
+
+ if (zoneDescriptor == null) {
+ return nullCompletedFuture();
+ }
+
+ dataNodesHistoryVolatile.put(zoneId, history);
+ } else if (e.tombstone()) {
+ dataNodesHistoryVolatile.remove(zoneId);
+ }
+
+ return nullCompletedFuture();
+ }
+
+ private static CompletableFuture<Void> processWatchEvent(
+ WatchEvent event,
+ byte[] keyPrefix,
+ BiFunction<Integer, Entry, CompletableFuture<Void>> processor
+ ) {
+ EntryEvent entryEvent = event.entryEvent();
+
+ Entry e = entryEvent.newEntry();
+
+ if (e != null && !e.empty()) {
+ assert startsWith(e.key(), keyPrefix) : "Unexpected key: " + new String(e.key(), UTF_8);
+
+ int zoneId = extractZoneId(e.key(), keyPrefix);
+
+ return processor.apply(zoneId, e);
+ }
+
+ return nullCompletedFuture();
+ }
+
+ @TestOnly
+ public ZoneTimers zoneTimers(int zoneId) {
+ return zoneTimers.computeIfAbsent(zoneId, k -> new ZoneTimers(zoneId, executor));
+ }
+
+ /**
+ * Representation of zone timer schedule, visible for testing purposes.
+ */
+ @VisibleForTesting
+ public static class ZoneTimerSchedule {
+ final StripedScheduledThreadPoolExecutor executor;
+ final int zoneId;
+ private @Nullable ScheduledFuture<?> taskFuture;
+ private long delay;
+
+ ZoneTimerSchedule(int zoneId, StripedScheduledThreadPoolExecutor executor) {
+ this.zoneId = zoneId;
+ this.executor = executor;
+ }
+
+ synchronized void reschedule(long delayInSeconds, Runnable task) {
+ stopScheduledTask();
+
+ delay = delayInSeconds;
+
+ if (delayInSeconds >= 0) {
+ taskFuture = executor.schedule(task, delayInSeconds, SECONDS, zoneId);
+ }
+ }
+
+ synchronized void stopScheduledTask() {
+ if (taskFuture != null && delay > 0) {
+ taskFuture.cancel(false);
+ taskFuture = null;
+
+ delay = 0;
+ }
+ }
+
+ synchronized void stopTimer() {
+ if (taskFuture != null) {
+ taskFuture.cancel(false);
+ taskFuture = null;
+ }
+ }
+
+ @TestOnly
+ public synchronized boolean taskIsScheduled() {
+ return taskFuture != null;
+ }
+
+ @TestOnly
+ public synchronized boolean taskIsCancelled() {
+ return taskFuture == null;
+ }
+
+ @TestOnly
+ public synchronized boolean taskIsDone() {
+ return taskFuture != null && taskFuture.isDone();
+ }
+ }
+
+ /**
+ * Zone timers class, visible for testing purposes.
+ */
+ @VisibleForTesting
+ public static class ZoneTimers {
+ public final ZoneTimerSchedule scaleUp;
+ public final ZoneTimerSchedule scaleDown;
+ public final ZoneTimerSchedule partitionReset;
+
+ ZoneTimers(int zoneId, StripedScheduledThreadPoolExecutor executor) {
+ this.scaleUp = new ZoneTimerSchedule(zoneId, executor);
+ this.scaleDown = new ZoneTimerSchedule(zoneId, executor);
+ this.partitionReset = new ZoneTimerSchedule(zoneId, executor);
+ }
+
+ void stopAllTimers() {
+ scaleUp.stopTimer();
+ scaleDown.stopTimer();
+ partitionReset.stopTimer();
+ }
+ }
+
+ @FunctionalInterface
+ private interface DataNodeHistoryContextMetaStorageGetter {
+ /** The future may contain {@code null} result. */
+ CompletableFuture<DataNodesHistoryContext> get(List<ByteArray> keys);
+ }
+
+ /**
+ * This class is used to generalize the logic of distribution zone timers related to their scheduled closure, see
+ * {@link #applyTimerClosure}. The only difference in implementations is the name of timer that is used for logging,
+ * the meta storage key, fields of {@link ZoneTimers} used for scheduling, etc.
+ */
+ private interface ScheduledTimer {
+ /**
+ * Name of the timer, is used for logging.
+ */
+ String name();
+
+ void init(DistributionZoneTimer timer);
+
+ ByteArray metaStorageKey();
+
+ DistributionZoneTimer timerFromContext(DataNodesHistoryContext context);
+
+ void reschedule(long delayInSeconds, Runnable runnable);
+
+ DataNodesHistoryEntry recalculateDataNodes(DataNodesHistory dataNodesHistory, DistributionZoneTimer timer);
+
+ @Nullable
+ DistributionZoneTimer scaleUpTimerAfterApply();
+
+ @Nullable
+ DistributionZoneTimer scaleDownTimerAfterApply();
+ }
+
+ private class ScaleUpScheduledTimer implements ScheduledTimer {
+ final CatalogZoneDescriptor zone;
+
+ private ScaleUpScheduledTimer(CatalogZoneDescriptor zone) {
+ this.zone = zone;
+ }
+
+ @Override
+ public void init(DistributionZoneTimer timer) {
+ int zoneId = zone.id();
+
+ if (timer.equals(DEFAULT_TIMER)) {
+ // Stop the scheduled task if the timer is default.
+ zoneTimers.computeIfAbsent(zoneId, k -> createZoneTimers(zone.id())).scaleUp.stopScheduledTask();
+
+ return;
+ }
+
+ reschedule(delayInSeconds(timer.timeToTrigger()), applyTimerClosure(zone, this));
+ }
+
+ /**
+ * Name of the timer, is used for logging.
+ */
+ @Override
+ public String name() {
+ return "scale up timer";
+ }
+
+ @Override
+ public ByteArray metaStorageKey() {
+ return zoneScaleUpTimerKey(zone.id());
+ }
+
+ @Override
+ public DistributionZoneTimer timerFromContext(DataNodesHistoryContext context) {
+ return context.scaleUpTimer();
+ }
+
+ @Override
+ public void reschedule(long delayInSeconds, Runnable runnable) {
+ zoneTimers.computeIfAbsent(zone.id(), k -> createZoneTimers(zone.id())).scaleUp.reschedule(delayInSeconds, runnable);
+ }
+
+ @Override
+ public DataNodesHistoryEntry recalculateDataNodes(
+ DataNodesHistory dataNodesHistory,
+ DistributionZoneTimer timer
+ ) {
+ return currentDataNodes(timer.timeToTrigger(), dataNodesHistory, timer, DEFAULT_TIMER, zone);
+ }
+
+ @Override
+ public @Nullable DistributionZoneTimer scaleUpTimerAfterApply() {
+ return DEFAULT_TIMER;
+ }
+
+ @Override
+ public @Nullable DistributionZoneTimer scaleDownTimerAfterApply() {
+ return null;
+ }
+ }
+
+ private class ScaleDownScheduledTimer implements ScheduledTimer {
+ final CatalogZoneDescriptor zone;
+
+ private ScaleDownScheduledTimer(CatalogZoneDescriptor zone) {
+ this.zone = zone;
+ }
+
+ @Override
+ public void init(DistributionZoneTimer timer) {
+ int zoneId = zone.id();
+
+ if (timer.equals(DEFAULT_TIMER)) {
+ // Stop the scheduled task if the timer is default.
+ zoneTimers.computeIfAbsent(zoneId, k -> createZoneTimers(zone.id())).scaleDown.stopScheduledTask();
+
+ return;
+ }
+
+ reschedule(delayInSeconds(timer.timeToTrigger()), applyTimerClosure(zone, this));
+ }
+
+ /**
+ * Name of the timer, is used for logging.
+ */
+ @Override
+ public String name() {
+ return "scale down timer";
+ }
+
+ @Override
+ public ByteArray metaStorageKey() {
+ return zoneScaleDownTimerKey(zone.id());
+ }
+
+ @Override
+ public DistributionZoneTimer timerFromContext(DataNodesHistoryContext context) {
+ return context.scaleDownTimer();
+ }
+
+ @Override
+ public void reschedule(long delayInSeconds, Runnable runnable) {
+ zoneTimers.computeIfAbsent(zone.id(), k -> createZoneTimers(zone.id())).scaleDown.reschedule(delayInSeconds, runnable);
+ }
+
+ @Override
+ public DataNodesHistoryEntry recalculateDataNodes(
+ DataNodesHistory dataNodesHistory,
+ DistributionZoneTimer timer
+ ) {
+ return currentDataNodes(timer.timeToTrigger(), dataNodesHistory, DEFAULT_TIMER, timer, zone);
+ }
+
+ @Override
+ public @Nullable DistributionZoneTimer scaleUpTimerAfterApply() {
+ return null;
+ }
+
+ @Override
+ public @Nullable DistributionZoneTimer scaleDownTimerAfterApply() {
+ return DEFAULT_TIMER;
+ }
+ }
+}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesMapSerializer.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesMapSerializer.java
deleted file mode 100644
index eace2f8b9f1..00000000000
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DataNodesMapSerializer.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.distributionzones;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.io.IgniteDataInput;
-import org.apache.ignite.internal.util.io.IgniteDataOutput;
-import org.apache.ignite.internal.versioned.VersionedSerialization;
-import org.apache.ignite.internal.versioned.VersionedSerializer;
-
-/**
- * {@link VersionedSerializer} for data nodes' maps.
- */
-public class DataNodesMapSerializer extends VersionedSerializer<Map<Node, Integer>> {
- /** Serializer instance. */
- public static final DataNodesMapSerializer INSTANCE = new DataNodesMapSerializer();
-
- private final NodeSerializer nodeSerializer = NodeSerializer.INSTANCE;
-
- @Override
- protected void writeExternalData(Map<Node, Integer> map, IgniteDataOutput out) throws IOException {
- out.writeVarInt(map.size());
- for (Map.Entry<Node, Integer> entry : map.entrySet()) {
- nodeSerializer.writeExternal(entry.getKey(), out);
- out.writeVarInt(entry.getValue());
- }
- }
-
- @Override
- protected Map<Node, Integer> readExternalData(byte protoVer, IgniteDataInput in) throws IOException {
- int size = in.readVarIntAsInt();
-
- Map<Node, Integer> map = new HashMap<>(IgniteUtils.capacity(size));
- for (int i = 0; i < size; i++) {
- Node node = nodeSerializer.readExternal(in);
- int count = in.readVarIntAsInt();
-
- map.put(node, count);
- }
-
- return map;
- }
-
- /**
- * Serializes a map to bytes.
- *
- * @param map Map to serialize.
- */
- public static byte[] serialize(Map<Node, Integer> map) {
- return VersionedSerialization.toBytes(map, INSTANCE);
- }
-
- /**
- * Deserializes a map from bytes.
- *
- * @param bytes Bytes.
- */
- public static Map<Node, Integer> deserialize(byte[] bytes) {
- return VersionedSerialization.fromBytes(bytes, INSTANCE);
- }
-}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
index 625a522e47e..f3510e9e09f 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
@@ -19,14 +19,9 @@ package org.apache.ignite.internal.distributionzones;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.emptySet;
-import static java.util.Collections.unmodifiableSet;
import static java.util.concurrent.CompletableFuture.allOf;
import static java.util.concurrent.CompletableFuture.failedFuture;
-import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toSet;
-import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
-import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
import static org.apache.ignite.internal.catalog.descriptors.ConsistencyMode.HIGH_AVAILABILITY;
import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_ALTER;
import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_CREATE;
@@ -34,24 +29,10 @@ import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_DROP;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.PARTITION_DISTRIBUTION_RESET_TIMEOUT;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.PARTITION_DISTRIBUTION_RESET_TIMEOUT_DEFAULT_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.conditionForRecoverableStateChanges;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.conditionForZoneCreation;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.conditionForZoneRemoval;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.createZoneManagerExecutor;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deleteDataNodesAndTriggerKeys;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeLogicalTopologySet;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.extractChangeTriggerRevision;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.extractDataNodes;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.toDataNodesMap;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.triggerScaleUpScaleDownKeysCondition;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateDataNodesAndScaleDownTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateDataNodesAndScaleUpTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateDataNodesAndTriggerKeys;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.filterDataNodes;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateLogicalTopologyAndVersion;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateLogicalTopologyAndVersionAndClusterId;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneTopologyAugmentation;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLastHandledTopology;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyClusterIdKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
@@ -71,24 +52,17 @@ import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFu
import static org.apache.ignite.internal.util.ExceptionUtils.hasCauseOrSuppressed;
import static org.apache.ignite.internal.util.IgniteUtils.inBusyLock;
import static org.apache.ignite.internal.util.IgniteUtils.inBusyLockAsync;
-import static org.apache.ignite.internal.util.IgniteUtils.shutdownAndAwaitTermination;
import static org.apache.ignite.lang.ErrorGroups.Common.NODE_STOPPING_ERR;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.internal.catalog.CatalogManager;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
@@ -103,16 +77,15 @@ import org.apache.ignite.internal.cluster.management.topology.api.LogicalTopolog
import org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologySnapshot;
import org.apache.ignite.internal.configuration.SystemDistributedConfiguration;
import org.apache.ignite.internal.configuration.utils.SystemDistributedConfigurationPropertyHolder;
-import org.apache.ignite.internal.distributionzones.causalitydatanodes.CausalityDataNodesEngine;
import org.apache.ignite.internal.distributionzones.events.HaZoneTopologyUpdateEvent;
import org.apache.ignite.internal.distributionzones.events.HaZoneTopologyUpdateEventParams;
import org.apache.ignite.internal.distributionzones.exception.DistributionZoneNotFoundException;
import org.apache.ignite.internal.distributionzones.rebalance.DistributionZoneRebalanceEngine;
import org.apache.ignite.internal.distributionzones.utils.CatalogAlterZoneEventListener;
import org.apache.ignite.internal.event.AbstractEventProducer;
-import org.apache.ignite.internal.lang.ByteArray;
+import org.apache.ignite.internal.hlc.ClockService;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
import org.apache.ignite.internal.lang.IgniteInternalException;
-import org.apache.ignite.internal.lang.IgniteStringFormatter;
import org.apache.ignite.internal.lang.NodeStoppingException;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
@@ -123,16 +96,12 @@ import org.apache.ignite.internal.metastorage.EntryEvent;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
import org.apache.ignite.internal.metastorage.Revisions;
import org.apache.ignite.internal.metastorage.WatchListener;
-import org.apache.ignite.internal.metastorage.dsl.CompoundCondition;
import org.apache.ignite.internal.metastorage.dsl.Condition;
import org.apache.ignite.internal.metastorage.dsl.Iif;
import org.apache.ignite.internal.metastorage.dsl.Operation;
-import org.apache.ignite.internal.metastorage.dsl.SimpleCondition;
import org.apache.ignite.internal.metastorage.dsl.StatementResult;
import org.apache.ignite.internal.metastorage.dsl.Update;
import org.apache.ignite.internal.metastorage.exceptions.CompactedException;
-import org.apache.ignite.internal.thread.NamedThreadFactory;
-import org.apache.ignite.internal.thread.StripedScheduledThreadPoolExecutor;
import org.apache.ignite.internal.util.IgniteSpinBusyLock;
import org.jetbrains.annotations.TestOnly;
@@ -156,36 +125,14 @@ public class DistributionZoneManager extends
/** Logical topology service to track topology changes. */
private final LogicalTopologyService logicalTopologyService;
- /** Executor for scheduling tasks for scale up and scale down processes. */
- private final StripedScheduledThreadPoolExecutor executor;
-
- /**
- * Map with states for distribution zones. States are needed to track nodes that we want to add or remove from the data nodes,
- * schedule and stop scale up and scale down processes.
- */
- private final Map<Integer, ZoneState> zonesState = new ConcurrentHashMap<>();
+ private final DataNodesManager dataNodesManager;
/** Listener for a topology events. */
- private final LogicalTopologyEventListener topologyEventListener = new LogicalTopologyEventListener() {
- @Override
- public void onNodeJoined(LogicalNode joinedNode, LogicalTopologySnapshot newTopology) {
- updateLogicalTopologyInMetaStorage(newTopology);
- }
-
- @Override
- public void onNodeLeft(LogicalNode leftNode, LogicalTopologySnapshot newTopology) {
- updateLogicalTopologyInMetaStorage(newTopology);
- }
-
- @Override
- public void onTopologyLeap(LogicalTopologySnapshot newTopology) {
- updateLogicalTopologyInMetaStorage(newTopology);
- }
- };
+ private final LogicalTopologyEventListener topologyEventListener = new DistributionZoneManagerLogicalTopologyEventListener();
/**
* The logical topology mapped to the MS revision.
- * TODO: https://issues.apache.org/jira/browse/IGNITE-23561 get rid of this map (or properly clean up it).
+ * TODO: https://issues.apache.org/jira/browse/IGNITE-24608 get rid of this map (or properly clean up it).
*/
private final ConcurrentSkipListMap<Long, Set<NodeWithAttributes>> logicalTopologyByRevision = new ConcurrentSkipListMap<>();
@@ -204,15 +151,9 @@ public class DistributionZoneManager extends
/** Rebalance engine. */
private final DistributionZoneRebalanceEngine rebalanceEngine;
- /** Causality data nodes engine. */
- private final CausalityDataNodesEngine causalityDataNodesEngine;
-
/** Catalog manager. */
private final CatalogManager catalogManager;
- /** Executor for scheduling rebalances. */
- private final ScheduledExecutorService rebalanceScheduler;
-
/** Configuration of HA mode. */
private final SystemDistributedConfigurationPropertyHolder<Integer> partitionDistributionResetTimeoutConfiguration;
@@ -225,6 +166,7 @@ public class DistributionZoneManager extends
* @param logicalTopologyService Logical topology service.
* @param catalogManager Catalog manager.
* @param systemDistributedConfiguration System distributed configuration.
+ * @param clockService Clock service.
*/
public DistributionZoneManager(
String nodeName,
@@ -232,8 +174,8 @@ public class DistributionZoneManager extends
MetaStorageManager metaStorageManager,
LogicalTopologyService logicalTopologyService,
CatalogManager catalogManager,
- ScheduledExecutorService rebalanceScheduler,
- SystemDistributedConfiguration systemDistributedConfiguration
+ SystemDistributedConfiguration systemDistributedConfiguration,
+ ClockService clockService
) {
this.metaStorageManager = metaStorageManager;
this.logicalTopologyService = logicalTopologyService;
@@ -241,13 +183,6 @@ public class DistributionZoneManager extends
this.topologyWatchListener = createMetastorageTopologyListener();
- this.rebalanceScheduler = rebalanceScheduler;
-
- executor = createZoneManagerExecutor(
- Math.min(Runtime.getRuntime().availableProcessors() * 3, 20),
- NamedThreadFactory.create(nodeName, "dst-zones-scheduler", LOG)
- );
-
// It's safe to leak with partially initialised object here, because rebalanceEngine is only accessible through this or by
// meta storage notification thread that won't start before all components start.
//noinspection ThisEscapedInObjectConstruction
@@ -258,16 +193,6 @@ public class DistributionZoneManager extends
catalogManager
);
- //noinspection ThisEscapedInObjectConstruction
- causalityDataNodesEngine = new CausalityDataNodesEngine(
- busyLock,
- registry,
- metaStorageManager,
- zonesState,
- this,
- catalogManager
- );
-
partitionDistributionResetTimeoutConfiguration = new SystemDistributedConfigurationPropertyHolder<>(
systemDistributedConfiguration,
this::onUpdatePartitionDistributionResetBusy,
@@ -275,6 +200,16 @@ public class DistributionZoneManager extends
PARTITION_DISTRIBUTION_RESET_TIMEOUT_DEFAULT_VALUE,
Integer::parseInt
);
+
+ dataNodesManager = new DataNodesManager(
+ nodeName,
+ busyLock,
+ metaStorageManager,
+ catalogManager,
+ clockService,
+ this::fireTopologyReduceLocalEvent,
+ partitionDistributionResetTimeoutConfiguration::currentValue
+ );
}
@Override
@@ -295,7 +230,7 @@ public class DistributionZoneManager extends
long recoveryRevision = recoveryFinishFuture.join().revision();
- restoreGlobalStateFromLocalMetastorage(recoveryRevision);
+ restoreGlobalStateFromLocalMetaStorage(recoveryRevision);
// If Catalog manager is empty, it gets initialized asynchronously and at this moment the initialization might not complete,
// nevertheless everything works correctly.
@@ -306,8 +241,8 @@ public class DistributionZoneManager extends
int catalogVersion = catalogManager.latestCatalogVersion();
return allOf(
- createOrRestoreZonesStates(recoveryRevision, catalogVersion),
- restoreLogicalTopologyChangeEventAndStartTimers(recoveryRevision, catalogVersion)
+ restoreLogicalTopologyChangeEvent(recoveryRevision),
+ dataNodesManager.startAsync(catalogManager.catalog(catalogVersion).zones(), recoveryRevision)
).thenComposeAsync((notUsed) -> rebalanceEngine.startAsync(catalogVersion), componentContext.executor());
});
}
@@ -320,7 +255,7 @@ public class DistributionZoneManager extends
busyLock.block();
- zonesState.values().forEach(ZoneState::stopTimers);
+ dataNodesManager.stop();
rebalanceEngine.stop();
@@ -328,9 +263,6 @@ public class DistributionZoneManager extends
metaStorageManager.unregisterWatch(topologyWatchListener);
- shutdownAndAwaitTermination(executor, 10, SECONDS);
- shutdownAndAwaitTermination(rebalanceScheduler, 10, SECONDS);
-
return nullCompletedFuture();
}
@@ -353,43 +285,32 @@ public class DistributionZoneManager extends
* @return The future with data nodes for the zoneId.
*/
public CompletableFuture<Set<String>> dataNodes(long causalityToken, int catalogVersion, int zoneId) {
- return causalityDataNodesEngine.dataNodes(causalityToken, catalogVersion, zoneId);
- }
-
- private CompletableFuture<Void> onUpdateScaleUpBusy(AlterZoneEventParameters parameters) {
- int zoneId = parameters.zoneDescriptor().id();
-
- int newScaleUp = parameters.zoneDescriptor().dataNodesAutoAdjustScaleUp();
-
- long causalityToken = parameters.causalityToken();
+ if (causalityToken < 1) {
+ throw new IllegalArgumentException("causalityToken must be greater then zero [causalityToken=" + causalityToken + '"');
+ }
- if (newScaleUp == IMMEDIATE_TIMER_VALUE) {
- return saveDataNodesToMetaStorageOnScaleUp(zoneId, causalityToken);
+ if (catalogVersion < 0) {
+ throw new IllegalArgumentException("catalogVersion must be greater or equal to zero [catalogVersion=" + catalogVersion + '"');
}
- // It is safe to zonesTimers.get(zoneId) in term of NPE because meta storage notifications are one-threaded
- // and this map will be initialized on a manager start or with catalog notification
- ZoneState zoneState = zonesState.get(zoneId);
+ if (zoneId < 0) {
+ throw new IllegalArgumentException("zoneId cannot be a negative number [zoneId=" + zoneId + '"');
+ }
- if (newScaleUp != INFINITE_TIMER_VALUE) {
- Optional<Long> highestRevision = zoneState.highestRevision(true);
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(causalityToken);
- assert highestRevision.isEmpty() || causalityToken >= highestRevision.get() : IgniteStringFormatter.format(
- "Expected causalityToken that is greater or equal to already seen meta storage events: highestRevision={}, "
- + "causalityToken={}",
- highestRevision.orElse(null), causalityToken
- );
+ return dataNodesManager.dataNodes(zoneId, timestamp, catalogVersion);
+ }
- zoneState.rescheduleScaleUp(
- newScaleUp,
- () -> saveDataNodesToMetaStorageOnScaleUp(zoneId, causalityToken),
- zoneId
- );
- } else {
- zoneState.stopScaleUp();
- }
+ private CompletableFuture<Void> onUpdateScaleUpBusy(AlterZoneEventParameters parameters) {
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(parameters.causalityToken());
- return nullCompletedFuture();
+ return dataNodesManager.onAutoAdjustAlteration(
+ parameters.zoneDescriptor(),
+ timestamp,
+ parameters.previousDescriptor().dataNodesAutoAdjustScaleUp(),
+ parameters.previousDescriptor().dataNodesAutoAdjustScaleDown()
+ );
}
private void onUpdatePartitionDistributionResetBusy(
@@ -416,275 +337,50 @@ public class DistributionZoneManager extends
// It is safe to zoneState.entrySet in term of ConcurrentModification and etc. because meta storage notifications are one-threaded
// and this map will be initialized on a manager start or with catalog notification or with distribution configuration changes.
- for (Map.Entry<Integer, ZoneState> zoneStateEntry : zonesState.entrySet()) {
- int zoneId = zoneStateEntry.getKey();
- CatalogZoneDescriptor zoneDescriptor = catalogManager.activeCatalog(updateTimestamp).zone(zoneId);
+ for (CatalogZoneDescriptor zoneDescriptor : catalogManager.activeCatalog(updateTimestamp).zones()) {
+ int zoneId = zoneDescriptor.id();
- if (zoneDescriptor == null || zoneDescriptor.consistencyMode() != HIGH_AVAILABILITY) {
+ if (zoneDescriptor.consistencyMode() != HIGH_AVAILABILITY) {
continue;
}
- ZoneState zoneState = zoneStateEntry.getValue();
-
- if (partitionDistributionResetTimeoutSeconds != INFINITE_TIMER_VALUE) {
- Optional<Long> highestRevision = zoneState.highestRevision();
-
- assert highestRevision.isEmpty() || causalityToken >= highestRevision.get() : IgniteStringFormatter.format(
- "Expected causalityToken that is greater or equal to already seen meta storage events: highestRevision={}, "
- + "causalityToken={}",
- highestRevision.orElse(null), causalityToken
- );
-
- zoneState.reschedulePartitionDistributionReset(
- partitionDistributionResetTimeoutSeconds,
- () -> fireTopologyReduceLocalEvent(causalityToken, zoneId),
- zoneId
- );
- } else {
- zoneState.stopPartitionDistributionReset();
- }
+ dataNodesManager.onUpdatePartitionDistributionReset(
+ zoneId,
+ partitionDistributionResetTimeoutSeconds,
+ () -> fireTopologyReduceLocalEvent(causalityToken, zoneId)
+ );
}
}
private CompletableFuture<Void> onUpdateScaleDownBusy(AlterZoneEventParameters parameters) {
- int zoneId = parameters.zoneDescriptor().id();
-
- int newScaleDown = parameters.zoneDescriptor().dataNodesAutoAdjustScaleDown();
-
- long causalityToken = parameters.causalityToken();
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(parameters.causalityToken());
- if (newScaleDown == IMMEDIATE_TIMER_VALUE) {
- return saveDataNodesToMetaStorageOnScaleDown(zoneId, causalityToken);
- }
-
- // It is safe to zonesTimers.get(zoneId) in term of NPE because meta storage notifications are one-threaded
- // and this map will be initialized on a manager start or with catalog notification
- ZoneState zoneState = zonesState.get(zoneId);
-
- if (newScaleDown != INFINITE_TIMER_VALUE) {
- Optional<Long> highestRevision = zoneState.highestRevision(false);
-
- assert highestRevision.isEmpty() || causalityToken >= highestRevision.get() : IgniteStringFormatter.format(
- "Expected causalityToken that is greater or equal to already seen meta storage events: highestRevision={}, "
- + "causalityToken={}",
- highestRevision.orElse(null), causalityToken
- );
-
- zoneState.rescheduleScaleDown(
- newScaleDown,
- () -> saveDataNodesToMetaStorageOnScaleDown(zoneId, causalityToken),
- zoneId
- );
- } else {
- zoneState.stopScaleDown();
- }
-
- return nullCompletedFuture();
+ return dataNodesManager.onAutoAdjustAlteration(
+ parameters.zoneDescriptor(),
+ timestamp,
+ parameters.previousDescriptor().dataNodesAutoAdjustScaleUp(),
+ parameters.previousDescriptor().dataNodesAutoAdjustScaleDown()
+ );
}
- private CompletableFuture<Void> onUpdateFilter(AlterZoneEventParameters parameters) {
- int zoneId = parameters.zoneDescriptor().id();
+ private CompletableFuture<Void> onUpdateFilterBusy(AlterZoneEventParameters parameters) {
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(parameters.causalityToken());
- long causalityToken = parameters.causalityToken();
-
- return saveDataNodesToMetaStorageOnScaleUp(zoneId, causalityToken);
- }
-
- /**
- * Restores zones' states.
- *
- * @param zone Zone descriptor.
- * @param causalityToken Causality token.
- * @return Future reflecting the completion of creation or restoring a zone.
- */
- private CompletableFuture<Void> restoreZoneStateBusy(CatalogZoneDescriptor zone, long causalityToken) {
- int zoneId = zone.id();
+ Entry topologyEntry = metaStorageManager.getLocally(zonesLogicalTopologyKey(), parameters.causalityToken());
- Entry zoneDataNodesLocalMetaStorage = metaStorageManager.getLocally(zoneDataNodesKey(zoneId), causalityToken);
+ if (topologyEntry != null && topologyEntry.value() != null) {
+ Set<NodeWithAttributes> logicalTopology = deserializeLogicalTopologySet(topologyEntry.value());
- if (zoneDataNodesLocalMetaStorage.value() == null) {
- // In this case, creation of a zone was interrupted during restart.
- return onCreateZone(zone, causalityToken);
+ return dataNodesManager.onZoneFilterChange(parameters.zoneDescriptor(), timestamp, logicalTopology);
} else {
- Entry topologyAugmentationMapLocalMetaStorage = metaStorageManager.getLocally(zoneTopologyAugmentation(zoneId), causalityToken);
-
- ConcurrentSkipListMap<Long, Augmentation> topologyAugmentationMap;
-
- if (topologyAugmentationMapLocalMetaStorage.value() == null) {
- // This case means that there won't any logical topology updates before restart.
- topologyAugmentationMap = new ConcurrentSkipListMap<>();
- } else {
- topologyAugmentationMap = TopologyAugmentationMapSerializer.deserialize(topologyAugmentationMapLocalMetaStorage.value());
- }
-
- ZoneState zoneState = new ZoneState(executor, topologyAugmentationMap);
-
- ZoneState prevZoneState = zonesState.putIfAbsent(zoneId, zoneState);
-
- assert prevZoneState == null : "Zone's state was created twice [zoneId = " + zoneId + ']';
- }
-
- return nullCompletedFuture();
- }
-
- private CompletableFuture<Void> onCreateZone(CatalogZoneDescriptor zone, long causalityToken) {
- int zoneId = zone.id();
-
- ConcurrentSkipListMap<Long, Augmentation> topologyAugmentationMap = new ConcurrentSkipListMap<>();
-
- ZoneState zoneState = new ZoneState(executor, topologyAugmentationMap);
-
- ZoneState prevZoneState = zonesState.putIfAbsent(zoneId, zoneState);
-
- assert prevZoneState == null : "Zone's state was created twice [zoneId = " + zoneId + ']';
-
- Set<Node> dataNodes = logicalTopology(causalityToken).stream().map(NodeWithAttributes::node).collect(toSet());
-
- causalityDataNodesEngine.onCreateZoneState(causalityToken, zone);
-
- return initDataNodesAndTriggerKeysInMetaStorage(zoneId, causalityToken, dataNodes);
- }
-
- /**
- * Restores timers that were scheduled before a node's restart. Take the highest revision from the
- * {@link ZoneState#topologyAugmentationMap()}, schedule scale up/scale down timers.
- *
- * @param catalogVersion Catalog version.
- * @return Future that represents the pending completion of the operation.
- * For the immediate timers it will be completed when data nodes will be updated in Meta Storage.
- */
- private CompletableFuture<Void> restoreTimers(int catalogVersion) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- for (CatalogZoneDescriptor zone : catalogManager.catalog(catalogVersion).zones()) {
- ZoneState zoneState = zonesState.get(zone.id());
-
- // Max revision from the {@link ZoneState#topologyAugmentationMap()} for node joins.
- Optional<Long> maxScaleUpRevisionOptional = zoneState.highestRevision(true);
-
- // Max revision from the {@link ZoneState#topologyAugmentationMap()} for node removals.
- Optional<Long> maxScaleDownRevisionOptional = zoneState.highestRevision(false);
-
- maxScaleUpRevisionOptional.ifPresent(
- maxScaleUpRevision -> {
- // Take the highest revision from the topologyAugmentationMap and schedule scale up/scale down,
- // meaning that all augmentations of nodes will be taken into account in newly created timers.
- // If augmentations have already been proposed to data nodes in the metastorage before restart,
- // that means we have updated corresponding trigger key and it's value will be greater or equal to
- // the highest revision from the topologyAugmentationMap, and current timer won't affect data nodes.
-
- futures.add(scheduleTimers(zone, true, false, maxScaleUpRevision));
- }
- );
-
- maxScaleDownRevisionOptional.ifPresent(
- maxScaleDownRevision -> futures.add(scheduleTimers(zone, false, true, maxScaleDownRevision))
- );
- }
-
- return allOf(futures.toArray(CompletableFuture[]::new));
- }
-
- /**
- * Method initialise data nodes value for the specified zone, also sets {@code revision} to the
- * {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} and {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)}
- * if it passes the condition. It is called on the first creation of a zone.
- *
- * @param zoneId Unique id of a zone
- * @param revision Revision of an event that has triggered this method.
- * @param dataNodes Data nodes.
- * @return Future reflecting the completion of initialisation of zone's keys in meta storage.
- */
- private CompletableFuture<Void> initDataNodesAndTriggerKeysInMetaStorage(
- int zoneId,
- long revision,
- Set<Node> dataNodes
- ) {
- if (!busyLock.enterBusy()) {
- throw new IgniteInternalException(NODE_STOPPING_ERR, new NodeStoppingException());
- }
-
- try {
- // Update data nodes for a zone only if the corresponding data nodes keys weren't initialised in ms yet.
- CompoundCondition triggerKeyCondition = conditionForZoneCreation(zoneId);
-
- Update dataNodesAndTriggerKeyUpd = updateDataNodesAndTriggerKeys(
- zoneId,
- revision,
- DataNodesMapSerializer.serialize(toDataNodesMap(dataNodes))
- );
-
- Iif iif = iif(triggerKeyCondition, dataNodesAndTriggerKeyUpd, ops().yield(false));
-
- return metaStorageManager.invoke(iif)
- .thenApply(StatementResult::getAsBoolean)
- .whenComplete((invokeResult, e) -> {
- if (e != null) {
- LOG.error(
- "Failed to update zones' dataNodes value [zoneId = {}, dataNodes = {}, revision = {}]",
- e,
- zoneId,
- dataNodes,
- revision
- );
- } else if (invokeResult) {
- LOG.info("Update zones' dataNodes value [zoneId = {}, dataNodes = {}, revision = {}]",
- zoneId,
- dataNodes,
- revision
- );
- } else {
- LOG.debug(
- "Failed to update zones' dataNodes value [zoneId = {}, dataNodes = {}, revision = {}]",
- zoneId,
- dataNodes,
- revision
- );
- }
- }).thenCompose((ignored) -> nullCompletedFuture());
- } finally {
- busyLock.leaveBusy();
+ return nullCompletedFuture();
}
}
- /**
- * Method deletes data nodes value for the specified zone.
- *
- * @param zoneId Unique id of a zone
- * @param revision Revision of an event that has triggered this method.
- */
- private CompletableFuture<Void> removeTriggerKeysAndDataNodes(int zoneId, long revision) {
- if (!busyLock.enterBusy()) {
- throw new IgniteInternalException(NODE_STOPPING_ERR, new NodeStoppingException());
- }
+ private CompletableFuture<?> onCreateZone(CatalogZoneDescriptor zone, long causalityToken) {
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(causalityToken);
- try {
- SimpleCondition triggerKeyCondition = conditionForZoneRemoval(zoneId);
-
- Update removeKeysUpd = deleteDataNodesAndTriggerKeys(zoneId, revision);
-
- Iif iif = iif(triggerKeyCondition, removeKeysUpd, ops().yield(false));
-
- return metaStorageManager.invoke(iif)
- .thenApply(StatementResult::getAsBoolean)
- .whenComplete((invokeResult, e) -> {
- if (e != null) {
- LOG.error(
- "Failed to delete zone's dataNodes keys [zoneId = {}, revision = {}]",
- e,
- zoneId,
- revision
- );
- } else if (invokeResult) {
- LOG.info("Delete zone's dataNodes keys [zoneId = {}, revision = {}]", zoneId, revision);
- } else {
- LOG.debug("Failed to delete zone's dataNodes keys [zoneId = {}, revision = {}]", zoneId, revision);
- }
- })
- .thenCompose(ignored -> nullCompletedFuture());
- } finally {
- busyLock.leaveBusy();
- }
+ return dataNodesManager.onZoneCreate(zone.id(), timestamp, filterDataNodes(logicalTopology(causalityToken), zone));
}
/**
@@ -751,7 +447,7 @@ public class DistributionZoneManager extends
*
* @param recoveryRevision Revision of the Meta Storage after its recovery.
*/
- private void restoreGlobalStateFromLocalMetastorage(long recoveryRevision) {
+ private void restoreGlobalStateFromLocalMetaStorage(long recoveryRevision) {
Entry lastHandledTopologyEntry = metaStorageManager.getLocally(zonesLastHandledTopology(), recoveryRevision);
Entry nodeAttributesEntry = metaStorageManager.getLocally(zonesNodesAttributes(), recoveryRevision);
@@ -795,28 +491,36 @@ public class DistributionZoneManager extends
byte[] newLogicalTopologyBytes;
Set<NodeWithAttributes> newLogicalTopology = null;
+ Set<NodeWithAttributes> oldLogicalTopology = null;
- long revision = 0;
+ HybridTimestamp timestamp = evt.timestamp();
for (EntryEvent event : evt.entryEvents()) {
Entry e = event.newEntry();
+ Entry old = event.oldEntry();
- if (Arrays.equals(e.key(), zonesLogicalTopologyVersionKey().bytes())) {
- revision = e.revision();
- } else if (Arrays.equals(e.key(), zonesLogicalTopologyKey().bytes())) {
+ if (Arrays.equals(e.key(), zonesLogicalTopologyKey().bytes())) {
newLogicalTopologyBytes = e.value();
+ assert newLogicalTopologyBytes != null : "New topology is null.";
+
newLogicalTopology = deserializeLogicalTopologySet(newLogicalTopologyBytes);
+
+ byte[] oldLogicalTopologyBytes = old.value();
+
+ if (oldLogicalTopologyBytes != null) {
+ oldLogicalTopology = deserializeLogicalTopologySet(oldLogicalTopologyBytes);
+ }
}
}
assert newLogicalTopology != null : "The event doesn't contain logical topology";
- assert revision > 0 : "The event doesn't contain logical topology version";
- // It is safe to get the latest version of the catalog as we are in the metastore thread.
- int catalogVersion = catalogManager.latestCatalogVersion();
+ if (oldLogicalTopology == null) {
+ oldLogicalTopology = newLogicalTopology;
+ }
- return onLogicalTopologyUpdate(newLogicalTopology, revision, catalogVersion);
+ return onLogicalTopologyUpdate(newLogicalTopology, oldLogicalTopology, evt.revision(), timestamp);
} finally {
busyLock.leaveBusy();
}
@@ -829,106 +533,67 @@ public class DistributionZoneManager extends
/**
* Reaction on an update of logical topology. In this method {@link DistributionZoneManager#logicalTopology},
- * {@link DistributionZoneManager#nodesAttributes}, {@link ZoneState#topologyAugmentationMap} are updated.
+ * {@link DistributionZoneManager#nodesAttributes} are updated.
* This fields are saved to Meta Storage, also timers are scheduled.
* Note that all futures of Meta Storage updates that happen in this method are returned from this method.
*
* @param newLogicalTopology New logical topology.
- * @param revision Revision of the logical topology update.
- * @param catalogVersion Actual version of the Catalog.
+ * @param oldLogicalTopology Old logical topology.
+ * @param revision Revision of the event.
+ * @param timestamp Event timestamp.
* @return Future reflecting the completion of the actions needed when logical topology was updated.
*/
- private CompletableFuture<Void> onLogicalTopologyUpdate(Set<NodeWithAttributes> newLogicalTopology, long revision, int catalogVersion) {
- Set<NodeWithAttributes> currentLogicalTopology = logicalTopology(revision);
-
- Set<Node> removedNodes =
- currentLogicalTopology.stream()
- .filter(node -> !newLogicalTopology.contains(node))
- .map(NodeWithAttributes::node)
- .collect(toSet());
-
- Set<Node> addedNodes =
- newLogicalTopology.stream()
- .filter(node -> !currentLogicalTopology.contains(node))
- .map(NodeWithAttributes::node)
- .collect(toSet());
-
- Set<Integer> zoneIds = new HashSet<>();
+ private CompletableFuture<Void> onLogicalTopologyUpdate(
+ Set<NodeWithAttributes> newLogicalTopology,
+ Set<NodeWithAttributes> oldLogicalTopology,
+ long revision,
+ HybridTimestamp timestamp
+ ) {
+ logicalTopologyByRevision.put(revision, newLogicalTopology);
List<CompletableFuture<Void>> futures = new ArrayList<>();
- logicalTopologyByRevision.put(revision, newLogicalTopology);
+ int catalogVersion = catalogManager.activeCatalogVersion(timestamp.longValue());
for (CatalogZoneDescriptor zone : catalogManager.catalog(catalogVersion).zones()) {
- int zoneId = zone.id();
-
- updateLocalTopologyAugmentationMap(addedNodes, removedNodes, revision, zoneId);
-
- futures.add(scheduleTimers(zone, !addedNodes.isEmpty(), !removedNodes.isEmpty(), revision));
+ CompletableFuture<Void> f = dataNodesManager.onTopologyChange(
+ zone,
+ revision,
+ timestamp,
+ newLogicalTopology,
+ oldLogicalTopology
+ );
- zoneIds.add(zone.id());
+ futures.add(f);
}
newLogicalTopology.forEach(n -> nodesAttributes.put(n.nodeId(), n));
- futures.add(saveRecoverableStateToMetastorage(zoneIds, revision, newLogicalTopology));
+ futures.add(saveRecoverableStateToMetastorage(revision, newLogicalTopology));
return allOf(futures.toArray(CompletableFuture[]::new));
}
- /**
- * Update local topology augmentation map with newly added and removed nodes.
- *
- * @param addedNodes Nodes that was added to a topology and should be added to zones data nodes.
- * @param removedNodes Nodes that was removed from a topology and should be removed from zones data nodes.
- * @param revision Revision of the event that triggered this method.
- * @param zoneId Zone's id.
- */
- private void updateLocalTopologyAugmentationMap(Set<Node> addedNodes, Set<Node> removedNodes, long revision, int zoneId) {
- if (!addedNodes.isEmpty()) {
- zonesState.get(zoneId).nodesToAddToDataNodes(addedNodes, revision);
- }
-
- if (!removedNodes.isEmpty()) {
- zonesState.get(zoneId).nodesToRemoveFromDataNodes(removedNodes, revision);
- }
- }
-
/**
* Saves recoverable state of the Distribution Zone Manager to Meta Storage atomically in one batch.
* After restart it could be used to restore these fields.
*
- * @param zoneIds Set of zone id's, whose states will be saved in the Meta Storage.
* @param revision Revision of the event.
* @param newLogicalTopology New logical topology.
* @return Future representing pending completion of the operation.
*/
private CompletableFuture<Void> saveRecoverableStateToMetastorage(
- Set<Integer> zoneIds,
long revision,
Set<NodeWithAttributes> newLogicalTopology
) {
- Operation[] puts = new Operation[3 + zoneIds.size()];
-
- puts[0] = put(zonesNodesAttributes(), NodesAttributesSerializer.serialize(nodesAttributes()));
-
- puts[1] = put(zonesRecoverableStateRevision(), longToBytesKeepingOrder(revision));
-
- puts[2] = put(
- zonesLastHandledTopology(),
- LogicalTopologySetSerializer.serialize(newLogicalTopology)
- );
-
- int i = 3;
-
- // TODO: https://issues.apache.org/jira/browse/IGNITE-19491 Properly utilise topology augmentation map. Also this map
- // TODO: can be saved only once for all zones.
- for (Integer zoneId : zoneIds) {
- puts[i++] = put(
- zoneTopologyAugmentation(zoneId),
- TopologyAugmentationMapSerializer.serialize(zonesState.get(zoneId).topologyAugmentationMap())
- );
- }
+ Operation[] puts = {
+ put(zonesNodesAttributes(), NodesAttributesSerializer.serialize(nodesAttributes())),
+ put(zonesRecoverableStateRevision(), longToBytesKeepingOrder(revision)),
+ put(
+ zonesLastHandledTopology(),
+ LogicalTopologySetSerializer.serialize(newLogicalTopology)
+ )
+ };
Iif iif = iif(
conditionForRecoverableStateChanges(revision),
@@ -952,73 +617,7 @@ public class DistributionZoneManager extends
}
/**
- * Schedules scale up and scale down timers.
- *
- * @param zone Zone descriptor.
- * @param nodesAdded Flag indicating that nodes was added to a topology and should be added to zones data nodes.
- * @param nodesRemoved Flag indicating that nodes was removed from a topology and should be removed from zones data nodes.
- * @param revision Revision that triggered that event.
- * @return Future that represents the pending completion of the operation.
- * For the immediate timers it will be completed when data nodes will be updated in Meta Storage.
- */
- private CompletableFuture<Void> scheduleTimers(CatalogZoneDescriptor zone, boolean nodesAdded, boolean nodesRemoved, long revision) {
- int autoAdjust = zone.dataNodesAutoAdjust();
- int autoAdjustScaleDown = zone.dataNodesAutoAdjustScaleDown();
- int autoAdjustScaleUp = zone.dataNodesAutoAdjustScaleUp();
- int partitionReset = partitionDistributionResetTimeoutConfiguration.currentValue();
-
- int zoneId = zone.id();
-
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- if ((nodesAdded || nodesRemoved) && autoAdjust != INFINITE_TIMER_VALUE) {
- // TODO: IGNITE-18134 Create scheduler with dataNodesAutoAdjust timer.
- throw new UnsupportedOperationException("Data nodes auto adjust is not supported.");
- } else {
- if (nodesAdded) {
- if (autoAdjustScaleUp == IMMEDIATE_TIMER_VALUE) {
- futures.add(saveDataNodesToMetaStorageOnScaleUp(zoneId, revision));
- }
-
- if (autoAdjustScaleUp != INFINITE_TIMER_VALUE) {
- zonesState.get(zoneId).rescheduleScaleUp(
- autoAdjustScaleUp,
- () -> saveDataNodesToMetaStorageOnScaleUp(zoneId, revision),
- zoneId
- );
- }
- }
-
- if (nodesRemoved) {
- if (zone.consistencyMode() == HIGH_AVAILABILITY) {
- if (partitionReset != INFINITE_TIMER_VALUE) {
- zonesState.get(zoneId).reschedulePartitionDistributionReset(
- partitionReset,
- () -> fireTopologyReduceLocalEvent(revision, zoneId),
- zoneId
- );
- }
- }
-
- if (autoAdjustScaleDown == IMMEDIATE_TIMER_VALUE) {
- futures.add(saveDataNodesToMetaStorageOnScaleDown(zoneId, revision));
- }
-
- if (autoAdjustScaleDown != INFINITE_TIMER_VALUE) {
- zonesState.get(zoneId).rescheduleScaleDown(
- autoAdjustScaleDown,
- () -> saveDataNodesToMetaStorageOnScaleDown(zoneId, revision),
- zoneId
- );
- }
- }
- }
-
- return allOf(futures.toArray(CompletableFuture[]::new));
- }
-
- /**
- * Returns metastore long view of {@link org.apache.ignite.internal.hlc.HybridTimestamp} by revision.
+ * Returns metastore long view of {@link HybridTimestamp} by revision.
*
* @param revision Metastore revision.
* @return Appropriate metastore timestamp or -1 if revision is already compacted.
@@ -1033,7 +632,6 @@ public class DistributionZoneManager extends
return -1;
}
-
}
private void fireTopologyReduceLocalEvent(long revision, int zoneId) {
@@ -1048,530 +646,9 @@ public class DistributionZoneManager extends
});
}
- /**
- * Method updates data nodes value for the specified zone after scale up timer timeout, sets {@code revision} to the
- * {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} if it passes the condition.
- *
- * @param zoneId Unique id of a zone
- * @param revision Revision of an event that has triggered this method.
- */
- CompletableFuture<Void> saveDataNodesToMetaStorageOnScaleUp(int zoneId, long revision) {
- if (!busyLock.enterBusy()) {
- throw new IgniteInternalException(NODE_STOPPING_ERR, new NodeStoppingException());
- }
-
- try {
- ZoneState zoneState = zonesState.get(zoneId);
-
- if (zoneState == null) {
- // Zone was deleted
- return nullCompletedFuture();
- }
-
- Set<ByteArray> keysToGetFromMs = Set.of(
- zoneDataNodesKey(zoneId),
- zoneScaleUpChangeTriggerKey(zoneId),
- zoneScaleDownChangeTriggerKey(zoneId)
- );
-
- return metaStorageManager.getAll(keysToGetFromMs).thenCompose(values -> inBusyLock(busyLock, () -> {
- if (values.containsValue(null)) {
- // Zone was deleted
- return nullCompletedFuture();
- }
-
- Map<Node, Integer> dataNodesFromMetaStorage = extractDataNodes(values.get(zoneDataNodesKey(zoneId)));
-
- long scaleUpTriggerRevision = extractChangeTriggerRevision(values.get(zoneScaleUpChangeTriggerKey(zoneId)));
-
- long scaleDownTriggerRevision = extractChangeTriggerRevision(values.get(zoneScaleDownChangeTriggerKey(zoneId)));
-
- if (revision <= scaleUpTriggerRevision) {
- LOG.debug(
- "Revision of the event is less than the scale up revision from the metastorage "
- + "[zoneId = {}, revision = {}, scaleUpTriggerRevision = {}]",
- zoneId,
- revision,
- scaleUpTriggerRevision
- );
-
- return nullCompletedFuture();
- }
-
- List<Node> deltaToAdd = zoneState.nodesToBeAddedToDataNodes(scaleUpTriggerRevision, revision);
-
- Map<Node, Integer> newDataNodes = new HashMap<>(dataNodesFromMetaStorage);
-
- deltaToAdd.forEach(n -> newDataNodes.merge(n, 1, Integer::sum));
-
- // Update dataNodes, so nodeId will be updated with the latest seen data on the node.
- // For example, node could be restarted with new nodeId, we need to update it in the data nodes.
- deltaToAdd.forEach(n -> newDataNodes.put(n, newDataNodes.remove(n)));
-
- // Remove redundant nodes that are not presented in the data nodes.
- newDataNodes.entrySet().removeIf(e -> e.getValue() == 0);
-
- Update dataNodesAndTriggerKeyUpd = updateDataNodesAndScaleUpTriggerKey(
- zoneId,
- revision,
- DataNodesMapSerializer.serialize(newDataNodes)
- );
-
- Iif iif = iif(
- triggerScaleUpScaleDownKeysCondition(scaleUpTriggerRevision, scaleDownTriggerRevision, zoneId),
- dataNodesAndTriggerKeyUpd,
- ops().yield(false)
- );
-
- return metaStorageManager.invoke(iif)
- .thenApply(StatementResult::getAsBoolean)
- .thenCompose(invokeResult -> inBusyLock(busyLock, () -> {
- if (invokeResult) {
- // TODO: https://issues.apache.org/jira/browse/IGNITE-19491 Properly utilise this map
- // Currently we call clean up only on a node that successfully writes data nodes.
- LOG.info(
- "Updating data nodes for a zone after scale up has succeeded "
- + "[zoneId = {}, dataNodes = {}, revision = {}]",
- zoneId,
- newDataNodes,
- revision
- );
-
- zoneState.cleanUp(Math.min(scaleDownTriggerRevision, revision));
- } else {
- LOG.debug("Updating data nodes for a zone after scale up has not succeeded "
- + "[zoneId = {}, dataNodes = {}, revision = {}]",
- zoneId,
- newDataNodes,
- revision
- );
-
- return saveDataNodesToMetaStorageOnScaleUp(zoneId, revision);
- }
-
- return nullCompletedFuture();
- }));
- })).whenComplete((v, e) -> {
- if (e != null) {
- LOG.warn("Failed to update zones' dataNodes value after scale up [zoneId = {}, revision = {}]",
- e, zoneId, revision);
- }
- });
- } finally {
- busyLock.leaveBusy();
- }
- }
-
- /**
- * Method updates data nodes value for the specified zone after scale down timer timeout, sets {@code revision} to the
- * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} if it passes the condition.
- *
- * @param zoneId Unique id of a zone
- * @param revision Revision of an event that has triggered this method.
- */
- CompletableFuture<Void> saveDataNodesToMetaStorageOnScaleDown(int zoneId, long revision) {
- if (!busyLock.enterBusy()) {
- throw new IgniteInternalException(NODE_STOPPING_ERR, new NodeStoppingException());
- }
-
- try {
- ZoneState zoneState = zonesState.get(zoneId);
-
- if (zoneState == null) {
- // Zone was deleted
- return nullCompletedFuture();
- }
-
- Set<ByteArray> keysToGetFromMs = Set.of(
- zoneDataNodesKey(zoneId),
- zoneScaleUpChangeTriggerKey(zoneId),
- zoneScaleDownChangeTriggerKey(zoneId)
- );
-
- return metaStorageManager.getAll(keysToGetFromMs).thenCompose(values -> inBusyLock(busyLock, () -> {
- if (values.containsValue(null)) {
- // Zone was deleted
- return nullCompletedFuture();
- }
-
- Map<Node, Integer> dataNodesFromMetaStorage = extractDataNodes(values.get(zoneDataNodesKey(zoneId)));
-
- long scaleUpTriggerRevision = extractChangeTriggerRevision(values.get(zoneScaleUpChangeTriggerKey(zoneId)));
-
- long scaleDownTriggerRevision = extractChangeTriggerRevision(values.get(zoneScaleDownChangeTriggerKey(zoneId)));
-
- if (revision <= scaleDownTriggerRevision) {
- LOG.debug(
- "Revision of the event is less than the scale down revision from the metastorage "
- + "[zoneId = {}, revision = {}, scaleUpTriggerRevision = {}]",
- zoneId,
- revision,
- scaleDownTriggerRevision
- );
-
- return nullCompletedFuture();
- }
-
- List<Node> deltaToRemove = zoneState.nodesToBeRemovedFromDataNodes(scaleDownTriggerRevision, revision);
-
- Map<Node, Integer> newDataNodes = new HashMap<>(dataNodesFromMetaStorage);
-
- deltaToRemove.forEach(n -> newDataNodes.merge(n, -1, Integer::sum));
-
- // Remove redundant nodes that are not presented in the data nodes.
- newDataNodes.entrySet().removeIf(e -> e.getValue() == 0);
-
- Update dataNodesAndTriggerKeyUpd = updateDataNodesAndScaleDownTriggerKey(
- zoneId,
- revision,
- DataNodesMapSerializer.serialize(newDataNodes)
- );
-
- Iif iif = iif(
- triggerScaleUpScaleDownKeysCondition(scaleUpTriggerRevision, scaleDownTriggerRevision, zoneId),
- dataNodesAndTriggerKeyUpd,
- ops().yield(false)
- );
-
- return metaStorageManager.invoke(iif)
- .thenApply(StatementResult::getAsBoolean)
- .thenCompose(invokeResult -> inBusyLock(busyLock, () -> {
- if (invokeResult) {
- LOG.info(
- "Updating data nodes for a zone after scale down has succeeded "
- + "[zoneId = {}, dataNodes = {}, revision = {}]",
- zoneId,
- newDataNodes,
- revision
- );
-
- // TODO: https://issues.apache.org/jira/browse/IGNITE-19491 Properly utilise this map
- // Currently we call clean up only on a node that successfully writes data nodes.
- zoneState.cleanUp(Math.min(scaleUpTriggerRevision, revision));
- } else {
- LOG.debug("Updating data nodes for a zone after scale down has not succeeded "
- + "[zoneId = {}, dataNodes = {}, revision = {}]",
- zoneId,
- newDataNodes,
- revision
- );
-
- return saveDataNodesToMetaStorageOnScaleDown(zoneId, revision);
- }
-
- return nullCompletedFuture();
- }));
- })).whenComplete((v, e) -> {
- if (e != null) {
- LOG.warn("Failed to update zones' dataNodes value after scale down [zoneId = {}]", e, zoneId);
- }
- });
- } finally {
- busyLock.leaveBusy();
- }
- }
-
- /**
- * Class responsible for storing state for a distribution zone.
- * States are needed to track nodes that we want to add or remove from the data nodes,
- * to schedule and stop scale up and scale down processes.
- */
- public static class ZoneState {
- /** Schedule task for a scale up process. */
- private ScheduledFuture<?> scaleUpTask;
-
- /** Schedule task for a scale down process. */
- private ScheduledFuture<?> scaleDownTask;
-
- /** Schedule task for a partition distribution reset process. */
- private ScheduledFuture<?> partitionDistributionResetTask;
-
- /** The delay for the scale up task. */
- private long scaleUpTaskDelay;
-
- /** The delay for the scale down task. */
- private long scaleDownTaskDelay;
-
- /** The delay for the partition distribution reset task. */
- private long partitionDistributionResetTaskDelay;
-
- /**
- * Map that stores pairs revision -> {@link Augmentation} for a zone. With this map we can track which nodes
- * should be added or removed in the processes of scale up or scale down. Revision helps to track visibility of the events
- * of adding or removing nodes because any process of scale up or scale down has a revision that triggered this process.
- */
- private final ConcurrentSkipListMap<Long, Augmentation> topologyAugmentationMap;
-
- /** Executor for scheduling tasks for scale up and scale down processes. */
- private final StripedScheduledThreadPoolExecutor executor;
-
- /**
- * Constructor.
- *
- * @param executor Executor for scheduling tasks for scale up and scale down processes.
- */
- ZoneState(StripedScheduledThreadPoolExecutor executor) {
- this.executor = executor;
- topologyAugmentationMap = new ConcurrentSkipListMap<>();
- }
-
- /**
- * Constructor.
- *
- * @param executor Executor for scheduling tasks for scale up and scale down processes.
- * @param topologyAugmentationMap Map that stores pairs revision -> {@link Augmentation} for a zone. With this map we can
- * track which nodes should be added or removed in the processes of scale up or scale down. Revision helps to track
- * visibility of the events of adding or removing nodes because any process of scale up or scale down has a revision that
- * triggered this process.
- */
- ZoneState(StripedScheduledThreadPoolExecutor executor, ConcurrentSkipListMap<Long, Augmentation> topologyAugmentationMap) {
- this.executor = executor;
- this.topologyAugmentationMap = topologyAugmentationMap;
- }
-
- /**
- * Map that stores pairs revision -> {@link Augmentation} for a zone. With this map we can track which nodes
- * should be added or removed in the processes of scale up or scale down. Revision helps to track visibility of the events
- * of adding or removing nodes because any process of scale up or scale down has a revision that triggered this process.
- */
- public ConcurrentSkipListMap<Long, Augmentation> topologyAugmentationMap() {
- return topologyAugmentationMap;
- }
-
- /**
- * Reschedules existing scale up task, if it is not started yet and the delay of this task is not immediate,
- * or schedules new one, if the current task cannot be canceled.
- *
- * @param delay Delay to start runnable in seconds.
- * @param runnable Custom logic to run.
- * @param zoneId Unique id of a zone to determine the executor of the task.
- */
- public synchronized void rescheduleScaleUp(long delay, Runnable runnable, int zoneId) {
- stopScaleUp();
-
- scaleUpTask = executor.schedule(runnable, delay, SECONDS, zoneId);
-
- scaleUpTaskDelay = delay;
- }
-
- /**
- * Reschedules existing scale down task, if it is not started yet and the delay of this task is not immediate,
- * or schedules new one, if the current task cannot be canceled.
- *
- * @param delay Delay to start runnable in seconds.
- * @param runnable Custom logic to run.
- * @param zoneId Unique id of a zone to determine the executor of the task.
- */
- public synchronized void rescheduleScaleDown(long delay, Runnable runnable, int zoneId) {
- stopScaleDown();
-
- scaleDownTask = executor.schedule(runnable, delay, SECONDS, zoneId);
-
- scaleDownTaskDelay = delay;
- }
-
- /**
- * Reschedules existing partition distribution reset task, if it is not started yet and the delay of this task is not immediate,
- * or schedules new one, if the current task cannot be canceled.
- *
- * @param delay Delay to start runnable in seconds.
- * @param runnable Custom logic to run.
- * @param zoneId Unique id of a zone to determine the executor of the task.
- */
- public synchronized void reschedulePartitionDistributionReset(long delay, Runnable runnable, int zoneId) {
- stopPartitionDistributionReset();
-
- partitionDistributionResetTask = executor.schedule(runnable, delay, SECONDS, zoneId);
-
- partitionDistributionResetTaskDelay = delay;
- }
-
- /**
- * Cancels task for scale up and scale down. Used on {@link #onDropZoneBusy(DropZoneEventParameters)}.
- * Not need to check {@code scaleUpTaskDelay} and {@code scaleDownTaskDelay} because after timer stopping on zone delete event
- * the data nodes value will be updated.
- */
- synchronized void stopTimers() {
- if (scaleUpTask != null) {
- scaleUpTask.cancel(false);
- }
-
- if (scaleDownTask != null) {
- scaleDownTask.cancel(false);
- }
-
- if (partitionDistributionResetTask != null) {
- partitionDistributionResetTask.cancel(false);
- }
- }
-
- /**
- * Cancels task for scale up if it is not started yet and the delay of this task is not immediate.
- */
- synchronized void stopScaleUp() {
- if (scaleUpTask != null && scaleUpTaskDelay > 0) {
- scaleUpTask.cancel(false);
- }
- }
-
- /**
- * Cancels task for scale down if it is not started yet and the delay of this task is not immediate.
- */
- synchronized void stopScaleDown() {
- if (scaleDownTask != null && scaleDownTaskDelay > 0) {
- scaleDownTask.cancel(false);
- }
- }
-
- /**
- * Cancels task for partition distribution reset if it is not started yet and the delay of this task is not immediate.
- */
- synchronized void stopPartitionDistributionReset() {
- if (partitionDistributionResetTask != null && partitionDistributionResetTaskDelay > 0) {
- partitionDistributionResetTask.cancel(false);
- }
- }
-
- /**
- * Returns a set of nodes that should be added to zone's data nodes.
- *
- * @param scaleUpRevision Last revision of the scale up event. Nodes that were associated with this event
- * or with the lower revisions, won't be included in the accumulation.
- * @param revision Revision of the event for which this data nodes is needed.
- * Nodes that were associated with this event will be included.
- * @return List of nodes that should be added to zone's data nodes.
- */
- List<Node> nodesToBeAddedToDataNodes(long scaleUpRevision, long revision) {
- return accumulateNodes(scaleUpRevision, revision, true);
- }
-
- /**
- * Returns a set of nodes that should be removed from zone's data nodes.
- *
- * @param scaleDownRevision Last revision of the scale down event. Nodes that were associated with this event
- * or with the lower revisions, won't be included in the accumulation.
- * @param revision Revision of the event for which this data nodes is needed.
- * Nodes that were associated with this event will be included.
- * @return List of nodes that should be removed from zone's data nodes.
- */
- List<Node> nodesToBeRemovedFromDataNodes(long scaleDownRevision, long revision) {
- return accumulateNodes(scaleDownRevision, revision, false);
- }
-
- /**
- * Add nodes to the map where nodes that must be added to the zone's data nodes are accumulated.
- *
- * @param nodes Nodes to add to zone's data nodes.
- * @param revision Revision of the event that triggered this addition.
- */
- void nodesToAddToDataNodes(Set<Node> nodes, long revision) {
- topologyAugmentationMap.put(revision, new Augmentation(nodes, true));
- }
-
- /**
- * Add nodes to the map where nodes that must be removed from the zone's data nodes are accumulated.
- *
- * @param nodes Nodes to remove from zone's data nodes.
- * @param revision Revision of the event that triggered this addition.
- */
- void nodesToRemoveFromDataNodes(Set<Node> nodes, long revision) {
- topologyAugmentationMap.put(revision, new Augmentation(nodes, false));
- }
-
- /**
- * Accumulate nodes from the {@link ZoneState#topologyAugmentationMap} starting from the {@code fromKey} (excluding)
- * to the {@code toKey} (including), where flag {@code addition} indicates whether we should accumulate nodes that should be
- * added to data nodes, or removed.
- *
- * @param fromKey Starting key (excluding).
- * @param toKey Ending key (including).
- * @param addition Indicates whether we should accumulate nodes that should be added to data nodes, or removed.
- * @return Accumulated nodes.
- */
- private List<Node> accumulateNodes(long fromKey, long toKey, boolean addition) {
- return topologyAugmentationMap.subMap(fromKey, false, toKey, true).values()
- .stream()
- .filter(a -> a.addition == addition)
- .flatMap(a -> a.nodes.stream())
- .collect(toList());
- }
-
- /**
- * Cleans {@code topologyAugmentationMap} to the key, to which is safe to delete.
- * Safe means that this key was handled both by scale up and scale down schedulers.
- *
- * @param toKey Key in map to which is safe to delete data from {@code topologyAugmentationMap}.
- */
- private void cleanUp(long toKey) {
- topologyAugmentationMap.headMap(toKey, true).clear();
- }
-
- /**
- * Returns the highest revision which is presented in the {@link ZoneState#topologyAugmentationMap()} taking into account
- * the {@code addition} flag.
- *
- * @param addition Flag indicating the type of the nodes for which we want to find the highest revision.
- * @return The highest revision which is presented in the {@link ZoneState#topologyAugmentationMap()} taking into account
- * the {@code addition} flag.
- */
- Optional<Long> highestRevision(boolean addition) {
- return topologyAugmentationMap().entrySet()
- .stream()
- .filter(e -> e.getValue().addition == addition)
- .max(Map.Entry.comparingByKey())
- .map(Map.Entry::getKey);
- }
-
- /**
- * Returns the highest revision which is presented in the {@link ZoneState#topologyAugmentationMap()}.
- *
- * @return The highest revision which is presented in the {@link ZoneState#topologyAugmentationMap()}.
- */
- Optional<Long> highestRevision() {
- return topologyAugmentationMap().keySet()
- .stream()
- .max(Comparator.naturalOrder());
- }
-
- @TestOnly
- public synchronized ScheduledFuture<?> scaleUpTask() {
- return scaleUpTask;
- }
-
- @TestOnly
- public synchronized ScheduledFuture<?> scaleDownTask() {
- return scaleDownTask;
- }
-
- @TestOnly
- public synchronized ScheduledFuture<?> partitionDistributionResetTask() {
- return partitionDistributionResetTask;
- }
- }
-
- /**
- * Class stores the info about nodes that should be added or removed from the data nodes of a zone.
- * With flag {@code addition} we can track whether {@code nodeNames} should be added or removed.
- */
- public static class Augmentation {
- /** Nodes. */
- private final Set<Node> nodes;
-
- /** Flag that indicates whether {@code nodeNames} should be added or removed. */
- private final boolean addition;
-
- public Augmentation(Set<Node> nodes, boolean addition) {
- this.nodes = unmodifiableSet(nodes);
- this.addition = addition;
- }
-
- public boolean addition() {
- return addition;
- }
-
- public Set<Node> nodes() {
- return nodes;
- }
+ @TestOnly
+ public DataNodesManager dataNodesManager() {
+ return dataNodesManager;
}
/**
@@ -1584,11 +661,6 @@ public class DistributionZoneManager extends
return nodesAttributes;
}
- @TestOnly
- public Map<Integer, ZoneState> zonesState() {
- return zonesState;
- }
-
public Set<NodeWithAttributes> logicalTopology() {
return logicalTopology(Long.MAX_VALUE);
}
@@ -1620,44 +692,40 @@ public class DistributionZoneManager extends
catalogManager.listen(ZONE_ALTER, new ManagerCatalogAlterZoneEventListener());
}
- private CompletableFuture<Void> createOrRestoreZonesStates(long recoveryRevision, int catalogVersion) {
- List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- // TODO: IGNITE-20287 Clean up abandoned resources for dropped tables from vault and metastore
- for (CatalogZoneDescriptor zone : catalogManager.catalog(catalogVersion).zones()) {
- futures.add(restoreZoneStateBusy(zone, recoveryRevision));
- }
-
- return allOf(futures.toArray(CompletableFuture[]::new));
- }
-
/**
* Restore the event of the updating the logical topology from Meta Storage, that has not been completed before restart.
- * Also start scale up/scale down timers.
*
* @param recoveryRevision Revision of the Meta Storage after its recovery.
* @return Future that represents the pending completion of the operations.
*/
- private CompletableFuture<Void> restoreLogicalTopologyChangeEventAndStartTimers(long recoveryRevision, int catalogVersion) {
+ private CompletableFuture<Void> restoreLogicalTopologyChangeEvent(long recoveryRevision) {
Entry topologyEntry = metaStorageManager.getLocally(zonesLogicalTopologyKey(), recoveryRevision);
if (topologyEntry.value() != null) {
- Set<NodeWithAttributes> newLogicalTopology = deserializeLogicalTopologySet(topologyEntry.value());
+ Set<NodeWithAttributes> logicalTopology = deserializeLogicalTopologySet(topologyEntry.value());
long topologyRevision = topologyEntry.revision();
Entry lastUpdateRevisionEntry = metaStorageManager.getLocally(zonesRecoverableStateRevision(), recoveryRevision);
if (lastUpdateRevisionEntry.value() == null || topologyRevision > bytesToLongKeepingOrder(lastUpdateRevisionEntry.value())) {
- return onLogicalTopologyUpdate(newLogicalTopology, recoveryRevision, catalogVersion);
- } else {
- return restoreTimers(catalogVersion);
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(recoveryRevision);
+
+ return onLogicalTopologyUpdate(logicalTopology, logicalTopology, recoveryRevision, timestamp);
}
}
return nullCompletedFuture();
}
+ private CompletableFuture<?> onDropZoneBusy(DropZoneEventParameters parameters) {
+ long causalityToken = parameters.causalityToken();
+
+ HybridTimestamp timestamp = metaStorageManager.timestampByRevisionLocally(causalityToken);
+
+ return dataNodesManager.onZoneDrop(parameters.zoneId(), timestamp);
+ }
+
private class ManagerCatalogAlterZoneEventListener extends CatalogAlterZoneEventListener {
private ManagerCatalogAlterZoneEventListener() {
super(catalogManager);
@@ -1675,23 +743,24 @@ public class DistributionZoneManager extends
@Override
protected CompletableFuture<Void> onFilterUpdate(AlterZoneEventParameters parameters, String oldFilter) {
- return inBusyLock(busyLock, () -> onUpdateFilter(parameters));
+ return inBusyLock(busyLock, () -> onUpdateFilterBusy(parameters));
}
}
- private CompletableFuture<Void> onDropZoneBusy(DropZoneEventParameters parameters) {
- int zoneId = parameters.zoneId();
-
- long causalityToken = parameters.causalityToken();
-
- ZoneState zoneState = zonesState.get(zoneId);
-
- zoneState.stopTimers();
+ private class DistributionZoneManagerLogicalTopologyEventListener implements LogicalTopologyEventListener {
+ @Override
+ public void onNodeJoined(LogicalNode joinedNode, LogicalTopologySnapshot newTopology) {
+ updateLogicalTopologyInMetaStorage(newTopology);
+ }
- return removeTriggerKeysAndDataNodes(zoneId, causalityToken).thenRun(() -> {
- causalityDataNodesEngine.onDelete(causalityToken, zoneId);
+ @Override
+ public void onNodeLeft(LogicalNode leftNode, LogicalTopologySnapshot newTopology) {
+ updateLogicalTopologyInMetaStorage(newTopology);
+ }
- zonesState.remove(zoneId);
- });
+ @Override
+ public void onTopologyLeap(LogicalTopologySnapshot newTopology) {
+ updateLogicalTopologyInMetaStorage(newTopology);
+ }
}
}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneTimer.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneTimer.java
new file mode 100644
index 00000000000..c1bbee6b13b
--- /dev/null
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneTimer.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.Collections.unmodifiableSet;
+import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.nodeNames;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.ignite.internal.catalog.commands.CatalogUtils;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.util.io.IgniteDataInput;
+import org.apache.ignite.internal.util.io.IgniteDataOutput;
+import org.apache.ignite.internal.versioned.VersionedSerialization;
+import org.apache.ignite.internal.versioned.VersionedSerializer;
+
+/**
+ * Timer representation for distribution zones.
+ */
+public class DistributionZoneTimer {
+ /** Default timer. */
+ public static final DistributionZoneTimer DEFAULT_TIMER = new DistributionZoneTimer(HybridTimestamp.MIN_VALUE, 0, Set.of());
+
+ private final HybridTimestamp createTimestamp;
+
+ private final int timeToWaitInSeconds;
+
+ private final Set<NodeWithAttributes> nodes;
+
+ /**
+ * Constructor.
+ *
+ * @param createTimestamp Timestamp when the timer was created.
+ * @param timeToWaitInSeconds Time to wait in seconds. {@link CatalogUtils#INFINITE_TIMER_VALUE} is acceptable here.
+ * @param nodes Nodes that will be affected by the timer. This is a set of added nodes (for scale up timer) or removed node
+ * (for scale down timer).
+ */
+ public DistributionZoneTimer(
+ HybridTimestamp createTimestamp,
+ int timeToWaitInSeconds,
+ Set<NodeWithAttributes> nodes
+ ) {
+ this.createTimestamp = createTimestamp;
+ this.timeToWaitInSeconds = timeToWaitInSeconds;
+ this.nodes = nodes;
+ }
+
+ /**
+ * Timestamp when the timer will be applied.
+ *
+ * @return Timestamp.
+ */
+ public HybridTimestamp createTimestamp() {
+ return createTimestamp;
+ }
+
+ /**
+ * Time to wait in seconds.
+ *
+ * @return Time to wait in seconds. {@link CatalogUtils#INFINITE_TIMER_VALUE} is acceptable here.
+ */
+ public int timeToWaitInSeconds() {
+ return timeToWaitInSeconds;
+ }
+
+ /**
+ * Nodes that will be affected by the timer. This is a set of added nodes (for scale up timer) or removed node (for scale down timer).
+ *
+ * @return Set of nodes.
+ */
+ public Set<NodeWithAttributes> nodes() {
+ return unmodifiableSet(nodes);
+ }
+
+ /**
+ * Timestamp when the timer should be triggered.
+ *
+ * @return Timestamp to trigger.
+ */
+ public HybridTimestamp timeToTrigger() {
+ if (timeToWaitInSeconds == INFINITE_TIMER_VALUE) {
+ return HybridTimestamp.MAX_VALUE;
+ }
+
+ return createTimestamp.addPhysicalTime(timeToWaitInSeconds * 1000L);
+ }
+
+ /**
+ * Create a new timer with the same timestamp and nodes, but with a new time to wait.
+ *
+ * @param newTimeToWaitInSeconds New time to wait in seconds.
+ * @return New timer.
+ */
+ public DistributionZoneTimer modifyTimeToWait(int newTimeToWaitInSeconds) {
+ if (equals(DEFAULT_TIMER)) {
+ return DEFAULT_TIMER;
+ } else {
+ return new DistributionZoneTimer(createTimestamp, newTimeToWaitInSeconds, nodes);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DistributionZoneTimer that = (DistributionZoneTimer) o;
+ return timeToWaitInSeconds == that.timeToWaitInSeconds && Objects.equals(createTimestamp, that.createTimestamp)
+ && Objects.equals(nodes, that.nodes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(createTimestamp, timeToWaitInSeconds, nodes);
+ }
+
+ @Override
+ public String toString() {
+ String timeToWaitStr = (timeToWaitInSeconds == INFINITE_TIMER_VALUE ? "[infinite]" : String.valueOf(timeToWaitInSeconds));
+
+ return equals(DEFAULT_TIMER) ? "[empty]" : "[timestamp=" + createTimestamp
+ + ", timeToWaitInSeconds=" + timeToWaitStr
+ + ", nodes=" + nodeNames(nodes) + ']';
+ }
+
+ /**
+ * Serializer.
+ */
+ public static class DistributionZoneTimerSerializer extends VersionedSerializer<DistributionZoneTimer> {
+ /** Serializer instance. */
+ private static final DistributionZoneTimerSerializer INSTANCE = new DistributionZoneTimerSerializer();
+
+ @Override
+ protected void writeExternalData(DistributionZoneTimer object, IgniteDataOutput out) throws IOException {
+ // Using new TreeSet() because the order of elements is important: the serialized representation is compared during
+ // MetaStorageManager#invoke().
+ out.writeVarInt(object.createTimestamp.longValue());
+ out.writeVarInt(object.timeToWaitInSeconds);
+ out.writeCollection(new TreeSet<>(object.nodes), NodeWithAttributesSerializer.INSTANCE::writeExternal);
+ }
+
+ @Override
+ protected DistributionZoneTimer readExternalData(byte protoVer, IgniteDataInput in) throws IOException {
+ HybridTimestamp timestamp = HybridTimestamp.hybridTimestamp(in.readVarInt());
+ int timeToWaitInSeconds = in.readVarIntAsInt();
+ Set<NodeWithAttributes> nodes = in.readCollection(
+ unused -> new TreeSet<>(),
+ NodeWithAttributesSerializer.INSTANCE::readExternal
+ );
+
+ return new DistributionZoneTimer(timestamp, timeToWaitInSeconds, nodes);
+ }
+
+ public static byte[] serialize(DistributionZoneTimer timer) {
+ return VersionedSerialization.toBytes(timer, INSTANCE);
+ }
+
+ public static DistributionZoneTimer deserialize(byte[] bytes) {
+ return VersionedSerialization.fromBytes(bytes, INSTANCE);
+ }
+ }
+}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
index 7370db95f5f..9e3e58f3e61 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
@@ -17,23 +17,18 @@
package org.apache.ignite.internal.distributionzones;
-import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.DEFAULT_FILTER;
-import static org.apache.ignite.internal.metastorage.dsl.Conditions.and;
-import static org.apache.ignite.internal.metastorage.dsl.Conditions.exists;
import static org.apache.ignite.internal.metastorage.dsl.Conditions.notExists;
-import static org.apache.ignite.internal.metastorage.dsl.Conditions.notTombstone;
import static org.apache.ignite.internal.metastorage.dsl.Conditions.or;
import static org.apache.ignite.internal.metastorage.dsl.Conditions.value;
import static org.apache.ignite.internal.metastorage.dsl.Operations.ops;
import static org.apache.ignite.internal.metastorage.dsl.Operations.put;
-import static org.apache.ignite.internal.metastorage.dsl.Operations.remove;
-import static org.apache.ignite.internal.util.ByteUtils.bytesToLongKeepingOrder;
import static org.apache.ignite.internal.util.ByteUtils.longToBytesKeepingOrder;
import static org.apache.ignite.internal.util.ByteUtils.uuidToBytes;
+import static org.apache.ignite.internal.util.IgniteUtils.startsWith;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
@@ -41,7 +36,7 @@ import com.jayway.jsonpath.Option;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -54,12 +49,13 @@ import org.apache.ignite.internal.catalog.descriptors.CatalogStorageProfileDescr
import org.apache.ignite.internal.catalog.descriptors.CatalogTableDescriptor;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologySnapshot;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.ZoneState;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.distributionzones.DistributionZoneTimer.DistributionZoneTimerSerializer;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
import org.apache.ignite.internal.lang.ByteArray;
import org.apache.ignite.internal.metastorage.Entry;
import org.apache.ignite.internal.metastorage.dsl.CompoundCondition;
import org.apache.ignite.internal.metastorage.dsl.Operation;
-import org.apache.ignite.internal.metastorage.dsl.SimpleCondition;
import org.apache.ignite.internal.metastorage.dsl.Update;
import org.apache.ignite.internal.thread.NamedThreadFactory;
import org.apache.ignite.internal.thread.StripedScheduledThreadPoolExecutor;
@@ -74,21 +70,32 @@ public class DistributionZonesUtil {
private static final String DISTRIBUTION_ZONE_PREFIX = "distributionZone.";
/** Key prefix for zone's data nodes and trigger keys. */
- private static final String DISTRIBUTION_ZONE_DATA_NODES_PREFIX = "distributionZone.dataNodes.";
+ private static final String DISTRIBUTION_ZONE_DATA_NODES_PREFIX = DISTRIBUTION_ZONE_PREFIX + "dataNodes.";
- /** Key prefix for zone's data nodes. */
- public static final String DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX = DISTRIBUTION_ZONE_DATA_NODES_PREFIX + "value.";
+ /** Key prefix for zone's data nodes history. */
+ public static final String DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX = DISTRIBUTION_ZONE_DATA_NODES_PREFIX + "history.";
- public static final byte[] DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES =
- DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX.getBytes(StandardCharsets.UTF_8);
+ /** Key prefix for zone's data nodes history, in {@code byte[]} representation. */
+ public static final byte[] DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES =
+ DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX.getBytes(StandardCharsets.UTF_8);
- /** Key prefix for zone's scale up change trigger key. */
- private static final String DISTRIBUTION_ZONE_SCALE_UP_CHANGE_TRIGGER_PREFIX =
- DISTRIBUTION_ZONE_DATA_NODES_PREFIX + "scaleUpChangeTrigger.";
+ /** Key prefix for zone's scale up timer. */
+ public static final String DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX = DISTRIBUTION_ZONE_DATA_NODES_PREFIX + "scaleUpTimer.";
- /** Key prefix for zone's scale down change trigger key. */
- private static final String DISTRIBUTION_ZONE_SCALE_DOWN_CHANGE_TRIGGER_PREFIX =
- DISTRIBUTION_ZONE_DATA_NODES_PREFIX + "scaleDownChangeTrigger.";
+ /** Key prefix for zone's scale up timer, in {@code byte[]} representation. */
+ static final byte[] DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX_BYTES =
+ DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX.getBytes(StandardCharsets.UTF_8);
+
+ /** Key prefix for zone's scale down timer. */
+ public static final String DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX = DISTRIBUTION_ZONE_DATA_NODES_PREFIX + "scaleDownTimer.";
+
+ /** Key prefix for zone's scale down timer, in {@code byte[]} representation. */
+ static final byte[] DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX_BYTES =
+ DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX.getBytes(StandardCharsets.UTF_8);
+
+ /** Key prefix for zone's partition reset timer. */
+ private static final String DISTRIBUTION_ZONE_PARTITION_RESET_TIMER_PREFIX = DISTRIBUTION_ZONE_DATA_NODES_PREFIX
+ + "partitionResetTimer.";
/** Key prefix for zones' logical topology nodes and logical topology version. */
private static final String DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX = "distributionZones.logicalTopology.";
@@ -105,19 +112,12 @@ public class DistributionZonesUtil {
/** Key prefix for zones' logical topology nodes. */
private static final String DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY = DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX + "nodes";
- /** Key prefix for zones' configurations in vault. */
- private static final String DISTRIBUTION_ZONES_VERSIONED_CONFIGURATION_VAULT = "vault." + DISTRIBUTION_ZONE_PREFIX
- + "versionedConfiguration.";
-
/** Key prefix for zones' logical topology version. */
private static final String DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_VERSION = DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX + "version";
/** Key prefix for zones' logical topology cluster ID. */
private static final String DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_CLUSTER_ID = DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX + "clusterId";
- /** Key prefix that represents {@link ZoneState#topologyAugmentationMap()}.*/
- private static final String DISTRIBUTION_ZONES_TOPOLOGY_AUGMENTATION_PREFIX = "distributionZones.topologyAugmentation.";
-
/** ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY}. */
private static final ByteArray DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_KEY = new ByteArray(DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY);
@@ -140,18 +140,6 @@ public class DistributionZonesUtil {
private static final ByteArray DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_CLUSTER_ID_KEY =
new ByteArray(DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_CLUSTER_ID);
- /**
- * The initial value of trigger revision in case when it is not initialized in the meta storage.
- * It is possible because invoke to metastorage with the initialisation is async, and scale up/down propagation could be
- * propagated first. Initial value is -1, because for default zone, we initialise trigger keys with metastorage's applied revision,
- * which is 0 on a start.
- */
- private static final long INITIAL_TRIGGER_REVISION_VALUE = -1;
-
- /** ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_DATA_NODES_PREFIX}. */
- private static final ByteArray DISTRIBUTION_ZONES_DATA_NODES_KEY =
- new ByteArray(DISTRIBUTION_ZONE_DATA_NODES_PREFIX);
-
/**
* Internal property that determines partition group members reset timeout after the partition group majority loss.
*
@@ -173,57 +161,79 @@ public class DistributionZonesUtil {
public static final int REBALANCE_RETRY_DELAY_DEFAULT = 200;
/**
- * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX}.
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX}.
+ *
+ * @return ByteArray representation.
+ */
+ public static ByteArray zoneDataNodesHistoryPrefix() {
+ return new ByteArray(DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX);
+ }
+
+ /**
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX}.
*
* @param zoneId Zone id.
* @return ByteArray representation.
*/
- public static ByteArray zoneDataNodesKey(int zoneId) {
- return new ByteArray(DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX + zoneId);
+ public static ByteArray zoneDataNodesHistoryKey(int zoneId) {
+ return new ByteArray(DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX + zoneId);
}
/**
- * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX}.
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX}.
*
* @return ByteArray representation.
*/
- public static ByteArray zoneDataNodesKey() {
- return new ByteArray(DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES);
+ public static ByteArray zoneScaleUpTimerPrefix() {
+ return new ByteArray(DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX);
}
/**
- * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONES_VERSIONED_CONFIGURATION_VAULT}.
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX}.
*
* @param zoneId Zone id.
* @return ByteArray representation.
*/
- public static ByteArray zoneVersionedConfigurationKey(int zoneId) {
- return new ByteArray(DISTRIBUTION_ZONES_VERSIONED_CONFIGURATION_VAULT + zoneId);
+ public static ByteArray zoneScaleUpTimerKey(int zoneId) {
+ return new ByteArray(DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX + zoneId);
}
/**
- * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX}.
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX}.
*
* @return ByteArray representation.
*/
- public static ByteArray zonesLogicalTopologyPrefix() {
- return new ByteArray(DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX);
+ public static ByteArray zoneScaleDownTimerPrefix() {
+ return new ByteArray(DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX);
+ }
+
+ /**
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX}.
+ *
+ * @param zoneId Zone id.
+ * @return ByteArray representation.
+ */
+ public static ByteArray zoneScaleDownTimerKey(int zoneId) {
+ return new ByteArray(DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX + zoneId);
}
/**
- * The key needed for processing an event about zone's data node propagation on scale up.
- * With this key we can be sure that event was triggered only once.
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONE_PARTITION_RESET_TIMER_PREFIX}.
+ *
+ * @param zoneId Zone id.
+ * @return ByteArray representation.
*/
- public static ByteArray zoneScaleUpChangeTriggerKey(int zoneId) {
- return new ByteArray(DISTRIBUTION_ZONE_SCALE_UP_CHANGE_TRIGGER_PREFIX + zoneId);
+ public static ByteArray zonePartitionResetTimerKey(int zoneId) {
+ return new ByteArray(DISTRIBUTION_ZONE_PARTITION_RESET_TIMER_PREFIX + zoneId);
}
/**
- * The key needed for processing an event about zone's data node propagation on scale down.
- * With this key we can be sure that event was triggered only once.
+ * ByteArray representation of {@link DistributionZonesUtil#DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX}.
+ *
+ * @return ByteArray representation.
*/
- public static ByteArray zoneScaleDownChangeTriggerKey(int zoneId) {
- return new ByteArray(DISTRIBUTION_ZONE_SCALE_DOWN_CHANGE_TRIGGER_PREFIX + zoneId);
+ public static ByteArray zonesLogicalTopologyPrefix() {
+ return new ByteArray(DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX);
}
/**
@@ -252,13 +262,6 @@ public class DistributionZonesUtil {
return DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_CLUSTER_ID_KEY;
}
- /**
- * The key prefix needed for processing an event about zone's data nodes.
- */
- public static ByteArray zonesDataNodesPrefix() {
- return DISTRIBUTION_ZONES_DATA_NODES_KEY;
- }
-
/**
* The key that represents nodes' attributes in Meta Storage.
*/
@@ -281,27 +284,6 @@ public class DistributionZonesUtil {
return DISTRIBUTION_ZONES_LAST_HANDLED_TOPOLOGY_KEY;
}
- /**
- * The key that represents {@link ZoneState#topologyAugmentationMap()} in the Meta Storage.
- */
- static ByteArray zoneTopologyAugmentation(int zoneId) {
- return new ByteArray(DISTRIBUTION_ZONES_TOPOLOGY_AUGMENTATION_PREFIX + zoneId);
- }
-
- /**
- * Condition for creating all data nodes' related keys in Meta Storage. Condition passes only when
- * {@link DistributionZonesUtil#zoneDataNodesKey(int)} not exists and not a tombstone in the Meta Storage.
- *
- * @param zoneId Distribution zone id
- * @return Update condition.
- */
- static CompoundCondition conditionForZoneCreation(int zoneId) {
- return and(
- notExists(zoneDataNodesKey(zoneId)),
- notTombstone(zoneDataNodesKey(zoneId))
- );
- }
-
/**
* Condition fot updating recoverable state of Distribution zone manager.
* Update only if the revision of the event is newer than value in that trigger key.
@@ -316,109 +298,6 @@ public class DistributionZonesUtil {
);
}
- /**
- * Condition for removing all data nodes' related keys in Meta Storage.
- *
- * @param zoneId Distribution zone id
- * @return Update condition.
- */
- static SimpleCondition conditionForZoneRemoval(int zoneId) {
- return exists(zoneDataNodesKey(zoneId));
- }
-
- /**
- * Condition for updating {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} key.
- * Update only if the revision of the event is newer than value in that trigger key.
- *
- * @param scaleUpTriggerRevision Trigger revision of scale up.
- * @param scaleDownTriggerRevision Trigger revision of scale down.
- * @param zoneId Zone id.
- * @return Update condition.
- */
- static CompoundCondition triggerScaleUpScaleDownKeysCondition(long scaleUpTriggerRevision, long scaleDownTriggerRevision, int zoneId) {
- SimpleCondition scaleUpCondition;
-
- if (scaleUpTriggerRevision != INITIAL_TRIGGER_REVISION_VALUE) {
- scaleUpCondition = value(zoneScaleUpChangeTriggerKey(zoneId)).eq(longToBytesKeepingOrder(scaleUpTriggerRevision));
- } else {
- scaleUpCondition = notExists(zoneScaleUpChangeTriggerKey(zoneId));
- }
-
- SimpleCondition scaleDownCondition;
-
- if (scaleDownTriggerRevision != INITIAL_TRIGGER_REVISION_VALUE) {
- scaleDownCondition = value(zoneScaleDownChangeTriggerKey(zoneId)).eq(longToBytesKeepingOrder(scaleDownTriggerRevision));
- } else {
- scaleDownCondition = notExists(zoneScaleDownChangeTriggerKey(zoneId));
- }
-
- return and(scaleUpCondition, scaleDownCondition);
- }
-
- /**
- * Updates data nodes value for a zone and set {@code revision} to {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)}.
- *
- * @param zoneId Distribution zone id
- * @param revision Revision of the event.
- * @param nodes Data nodes.
- * @return Update command for the meta storage.
- */
- static Update updateDataNodesAndScaleUpTriggerKey(int zoneId, long revision, byte[] nodes) {
- return ops(
- put(zoneDataNodesKey(zoneId), nodes),
- put(zoneScaleUpChangeTriggerKey(zoneId), longToBytesKeepingOrder(revision))
- ).yield(true);
- }
-
- /**
- * Updates data nodes value for a zone and set {@code revision} to {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)}.
- *
- * @param zoneId Distribution zone id
- * @param revision Revision of the event.
- * @param nodes Data nodes.
- * @return Update command for the meta storage.
- */
- static Update updateDataNodesAndScaleDownTriggerKey(int zoneId, long revision, byte[] nodes) {
- return ops(
- put(zoneDataNodesKey(zoneId), nodes),
- put(zoneScaleDownChangeTriggerKey(zoneId), longToBytesKeepingOrder(revision))
- ).yield(true);
- }
-
-
- /**
- * Updates data nodes value for a zone and set {@code revision} to {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} and
- * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)}.
- *
- * @param zoneId Distribution zone id
- * @param revision Revision of the event.
- * @param nodes Data nodes.
- * @return Update command for the meta storage.
- */
- static Update updateDataNodesAndTriggerKeys(int zoneId, long revision, byte[] nodes) {
- return ops(
- put(zoneDataNodesKey(zoneId), nodes),
- put(zoneScaleUpChangeTriggerKey(zoneId), longToBytesKeepingOrder(revision)),
- put(zoneScaleDownChangeTriggerKey(zoneId), longToBytesKeepingOrder(revision))
- ).yield(true);
- }
-
- /**
- * Deletes data nodes, {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)},
- * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} values for a zone.
- *
- * @param zoneId Distribution zone id
- * @param revision Revision of the event.
- * @return Update command for the meta storage.
- */
- static Update deleteDataNodesAndTriggerKeys(int zoneId, long revision) {
- return ops(
- remove(zoneDataNodesKey(zoneId)),
- remove(zoneScaleUpChangeTriggerKey(zoneId)),
- remove(zoneScaleDownChangeTriggerKey(zoneId))
- ).yield(true);
- }
-
/**
* Updates logical topology and its version values for zones.
*
@@ -474,64 +353,78 @@ public class DistributionZonesUtil {
}
/**
- * Returns a map from a set of data nodes. This map has the following structure: node is mapped to integer,
- * integer represents how often node joined or leaved topology. In this case, set of nodes is interpreted as nodes
- * that joined topology, so all mappings will be node -> 1.
+ * Returns a set of data nodes retrieved from data nodes map, which value is more than 0.
*
- * @param dataNodes Set of data nodes.
- * @return Returns a map from a set of data nodes.
+ * @param dataNodes Data nodes with attributes set.
+ * @return Returns a set of data nodes retrieved from data nodes with attributes set.
*/
- public static Map<Node, Integer> toDataNodesMap(Set<Node> dataNodes) {
- Map<Node, Integer> dataNodesMap = new HashMap<>();
-
- dataNodes.forEach(n -> dataNodesMap.merge(n, 1, Integer::sum));
-
- return dataNodesMap;
+ public static Set<Node> dataNodes(Set<NodeWithAttributes> dataNodes) {
+ return dataNodes.stream().map(NodeWithAttributes::node).collect(toSet());
}
+ /**
+ * Parse the data nodes from bytes.
+ *
+ * @param dataNodesBytes Data nodes bytes.
+ * @param timestamp Timestamp.
+ * @return Set of nodes.
+ */
@Nullable
- public static Set<Node> parseDataNodes(byte[] dataNodesBytes) {
- return dataNodesBytes == null ? null : dataNodes(deserializeDataNodesMap(dataNodesBytes));
- }
+ public static Set<NodeWithAttributes> parseDataNodes(byte[] dataNodesBytes, HybridTimestamp timestamp) {
+ if (dataNodesBytes == null) {
+ return null;
+ }
+
+ DataNodesHistory dataNodesHistory = DataNodesHistorySerializer.deserialize(dataNodesBytes);
- public static Map<Node, Integer> deserializeDataNodesMap(byte[] bytes) {
- return DataNodesMapSerializer.deserialize(bytes);
+ return dataNodesHistory.dataNodesForTimestamp(timestamp).dataNodes();
}
+ /**
+ * Deserialize the set of nodes with attributes from the given serialized bytes.
+ *
+ * @param bytes Serialized set of nodes with attributes.
+ * @return Set of nodes with attributes.
+ */
public static Set<NodeWithAttributes> deserializeLogicalTopologySet(byte[] bytes) {
return LogicalTopologySetSerializer.deserialize(bytes);
}
- public static Map<UUID, NodeWithAttributes> deserializeNodesAttributes(byte[] bytes) {
+ static Map<UUID, NodeWithAttributes> deserializeNodesAttributes(byte[] bytes) {
return NodesAttributesSerializer.deserialize(bytes);
}
/**
- * Returns data nodes from the meta storage entry or empty map if the value is null.
+ * Meta storage entries to {@link DataNodesHistoryContext}.
*
- * @param dataNodesEntry Meta storage entry with data nodes.
- * @return Data nodes.
+ * @param entries Entries.
+ * @return DataNodeHistoryContext.
*/
- static Map<Node, Integer> extractDataNodes(Entry dataNodesEntry) {
- if (!dataNodesEntry.empty()) {
- return deserializeDataNodesMap(dataNodesEntry.value());
- } else {
- return emptyMap();
+ @Nullable
+ public static DistributionZonesUtil.DataNodesHistoryContext dataNodeHistoryContextFromValues(Collection<Entry> entries) {
+ DataNodesHistory dataNodesHistory = null;
+ DistributionZoneTimer scaleUpTimer = null;
+ DistributionZoneTimer scaleDownTimer = null;
+
+ for (Entry e : entries) {
+ if (e.empty()) {
+ return null;
+ }
+
+ assert e != null && e.key() != null : "Unexpected entry: " + e;
+
+ byte[] v = e.tombstone() ? null : e.value();
+
+ if (startsWith(e.key(), DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES)) {
+ dataNodesHistory = v == null ? null : DataNodesHistorySerializer.deserialize(v);
+ } else if (startsWith(e.key(), DISTRIBUTION_ZONE_SCALE_UP_TIMER_PREFIX_BYTES)) {
+ scaleUpTimer = v == null ? null : DistributionZoneTimerSerializer.deserialize(v);
+ } else if (startsWith(e.key(), DISTRIBUTION_ZONE_SCALE_DOWN_TIMER_PREFIX_BYTES)) {
+ scaleDownTimer = v == null ? null : DistributionZoneTimerSerializer.deserialize(v);
+ }
}
- }
- /**
- * Returns a trigger revision from the meta storage entry or {@link INITIAL_TRIGGER_REVISION_VALUE} if the value is null.
- *
- * @param revisionEntry Meta storage entry with data nodes.
- * @return Revision.
- */
- static long extractChangeTriggerRevision(Entry revisionEntry) {
- if (!revisionEntry.empty()) {
- return bytesToLongKeepingOrder(revisionEntry.value());
- } else {
- return INITIAL_TRIGGER_REVISION_VALUE;
- }
+ return new DataNodesHistoryContext(dataNodesHistory, scaleUpTimer, scaleDownTimer);
}
/**
@@ -610,23 +503,18 @@ public class DistributionZonesUtil {
/**
* Filters {@code dataNodes} according to the provided filter and storage profiles from {@code zoneDescriptor}.
- * Nodes' attributes and storage profiles are taken from {@code nodesAttributes} map.
*
- * @param dataNodes Data nodes.
+ * @param dataNodes Data nodes with attributes.
* @param zoneDescriptor Zone descriptor.
- * @param nodesAttributes Nodes' attributes which used for filtering.
* @return Filtered data nodes.
*/
- public static Set<String> filterDataNodes(
- Set<Node> dataNodes,
- CatalogZoneDescriptor zoneDescriptor,
- Map<UUID, NodeWithAttributes> nodesAttributes
+ public static Set<NodeWithAttributes> filterDataNodes(
+ Set<NodeWithAttributes> dataNodes,
+ CatalogZoneDescriptor zoneDescriptor
) {
-
return dataNodes.stream()
- .filter(n -> filterNodeAttributes(nodesAttributes.get(n.nodeId()).userAttributes(), zoneDescriptor.filter()))
- .filter(n -> filterStorageProfiles(nodesAttributes.get(n.nodeId()), zoneDescriptor.storageProfiles().profiles()))
- .map(Node::nodeName)
+ .filter(n -> filterNodeAttributes(n.userAttributes(), zoneDescriptor.filter()))
+ .filter(n -> filterStorageProfiles(n, zoneDescriptor.storageProfiles().profiles()))
.collect(toSet());
}
@@ -662,7 +550,6 @@ public class DistributionZonesUtil {
);
}
-
/**
* Returns list of table descriptors bound to the zone.
*
@@ -676,15 +563,84 @@ public class DistributionZonesUtil {
.collect(toList());
}
- /** Key prefix for zone's scale up change trigger key. */
- @TestOnly
- public static ByteArray zoneScaleUpChangeTriggerKeyPrefix() {
- return new ByteArray(DISTRIBUTION_ZONE_SCALE_UP_CHANGE_TRIGGER_PREFIX);
+ public static Set<String> nodeNames(Set<NodeWithAttributes> nodes) {
+ return nodes.stream().map(NodeWithAttributes::nodeName).collect(toSet());
}
- /** Key prefix for zone's scale down change trigger key. */
- @TestOnly
- public static ByteArray zoneScaleDownChangeTriggerKeyPrefix() {
- return new ByteArray(DISTRIBUTION_ZONE_SCALE_DOWN_CHANGE_TRIGGER_PREFIX);
+ /**
+ * Class representing data nodes' related values in Meta Storage.
+ */
+ public static class DataNodesHistoryContext {
+ @Nullable
+ private final DataNodesHistory dataNodesHistory;
+
+ @Nullable
+ private final DistributionZoneTimer scaleUpTimer;
+
+ @Nullable
+ private final DistributionZoneTimer scaleDownTimer;
+
+ DataNodesHistoryContext(
+ @Nullable DataNodesHistory dataNodesHistory,
+ @Nullable DistributionZoneTimer scaleUpTimer,
+ @Nullable DistributionZoneTimer scaleDownTimer
+ ) {
+ this.dataNodesHistory = dataNodesHistory;
+ this.scaleUpTimer = scaleUpTimer;
+ this.scaleDownTimer = scaleDownTimer;
+ }
+
+ /**
+ * Data nodes history.
+ *
+ * @return Data nodes history.
+ */
+ public DataNodesHistory dataNodesHistory() {
+ assert dataNodesHistory != null : "Data nodes history were not initialized.";
+
+ return dataNodesHistory;
+ }
+
+ /**
+ * Scale up timer.
+ *
+ * @return Scale up timer.
+ */
+ public DistributionZoneTimer scaleUpTimer() {
+ assert scaleUpTimer != null : "Scale up timer was not initialized.";
+
+ return scaleUpTimer;
+ }
+
+ /**
+ * Scale up timer present.
+ *
+ * @return Scale up timer present.
+ */
+ @TestOnly
+ public boolean scaleUpTimerPresent() {
+ return scaleUpTimer != null;
+ }
+
+ /**
+ * Scale down timer present.
+ *
+ * @return Scale down timer present.
+ */
+ @TestOnly
+ public boolean scaleDownTimerPresent() {
+ return scaleDownTimer != null;
+ }
+
+ /**
+ * Scale down timer.
+ *
+ * @return Scale down timer.
+ */
+ public DistributionZoneTimer scaleDownTimer() {
+ assert scaleDownTimer != null : "Scale down timer was not initialized.";
+
+ return scaleDownTimer;
+ }
}
}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/Node.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/Node.java
index cff9ef044a5..9f2d842d8d4 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/Node.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/Node.java
@@ -26,7 +26,7 @@ import org.apache.ignite.internal.tostring.S;
* {@code nodeId} that is unique identifier of a node, that changes after a restart.
* {@code nodeId} is needed to get node's attributes from the local state of the distribution zone manager.
*/
-public class Node {
+public class Node implements Comparable<Node> {
private final String nodeName;
private final UUID nodeId;
@@ -68,4 +68,15 @@ public class Node {
public String toString() {
return S.toString(Node.class, this);
}
+
+ @Override
+ public int compareTo(Node o) {
+ int compareConsistentId = nodeName.compareTo(o.nodeName);
+
+ if (compareConsistentId != 0) {
+ return compareConsistentId;
+ } else {
+ return nodeId.compareTo(o.nodeId);
+ }
+ }
}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/NodeWithAttributes.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/NodeWithAttributes.java
index 72e0f173b33..77096c473a7 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/NodeWithAttributes.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/NodeWithAttributes.java
@@ -27,9 +27,10 @@ import org.jetbrains.annotations.Nullable;
/**
* Structure that represents node with the attributes and which we store in Meta Storage when we store logical topology.
- * Light-weighted version of the {@link LogicalNode}.
+ * Light-weighted version of the {@link LogicalNode}. This class has to be comparable because it is used into ordered collections,
+ * which are serialized and their byte representation is compared on equality.
*/
-public class NodeWithAttributes {
+public class NodeWithAttributes implements Comparable<NodeWithAttributes> {
private final Node node;
@IgniteToStringInclude
@@ -113,4 +114,9 @@ public class NodeWithAttributes {
public String toString() {
return S.toString(NodeWithAttributes.class, this);
}
+
+ @Override
+ public int compareTo(NodeWithAttributes o) {
+ return node.compareTo(o.node);
+ }
}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/TopologyAugmentationMapSerializer.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/TopologyAugmentationMapSerializer.java
deleted file mode 100644
index c403137227d..00000000000
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/TopologyAugmentationMapSerializer.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.distributionzones;
-
-import java.io.IOException;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentSkipListMap;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.Augmentation;
-import org.apache.ignite.internal.util.io.IgniteDataInput;
-import org.apache.ignite.internal.util.io.IgniteDataOutput;
-import org.apache.ignite.internal.versioned.VersionedSerialization;
-import org.apache.ignite.internal.versioned.VersionedSerializer;
-
-/**
- * {@link VersionedSerializer} for topology augmentation maps represented with {@code ConcurrentSkipListMap<Long, Augmentation>}.
- */
-public class TopologyAugmentationMapSerializer extends VersionedSerializer<ConcurrentSkipListMap<Long, Augmentation>> {
- /** Serializer instance. */
- public static final TopologyAugmentationMapSerializer INSTANCE = new TopologyAugmentationMapSerializer();
-
- private final AugmentationSerializer augmentationSerializer = AugmentationSerializer.INSTANCE;
-
- @Override
- protected void writeExternalData(ConcurrentSkipListMap<Long, Augmentation> map, IgniteDataOutput out) throws IOException {
- out.writeVarInt(map.size());
- for (Entry<Long, Augmentation> entry : map.entrySet()) {
- out.writeVarInt(entry.getKey());
- augmentationSerializer.writeExternal(entry.getValue(), out);
- }
- }
-
- @Override
- protected ConcurrentSkipListMap<Long, Augmentation> readExternalData(byte protoVer, IgniteDataInput in) throws IOException {
- int length = in.readVarIntAsInt();
-
- ConcurrentSkipListMap<Long, Augmentation> map = new ConcurrentSkipListMap<>();
- for (int i = 0; i < length; i++) {
- map.put(in.readVarInt(), augmentationSerializer.readExternal(in));
- }
-
- return map;
- }
-
- /**
- * Serializes a map to bytes.
- *
- * @param map Map to serialize.
- */
- public static byte[] serialize(ConcurrentSkipListMap<Long, Augmentation> map) {
- return VersionedSerialization.toBytes(map, INSTANCE);
- }
-
- /**
- * Deserializes a map from bytes.
- *
- * @param bytes Bytes.
- */
- public static ConcurrentSkipListMap<Long, Augmentation> deserialize(byte[] bytes) {
- return VersionedSerialization.fromBytes(bytes, INSTANCE);
- }
-}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/CausalityDataNodesEngine.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/CausalityDataNodesEngine.java
deleted file mode 100644
index 40f95279a30..00000000000
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/CausalityDataNodesEngine.java
+++ /dev/null
@@ -1,540 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.distributionzones.causalitydatanodes;
-
-import static java.lang.Math.max;
-import static java.util.Collections.emptySet;
-import static java.util.stream.Collectors.toSet;
-import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeLogicalTopologySet;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.filterDataNodes;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
-import static org.apache.ignite.internal.util.ByteUtils.bytesToLongKeepingOrder;
-import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
-import static org.apache.ignite.internal.util.IgniteUtils.inBusyLock;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import org.apache.ignite.internal.catalog.CatalogManager;
-import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
-import org.apache.ignite.internal.catalog.events.CatalogEventParameters;
-import org.apache.ignite.internal.causality.IncrementalVersionedValue;
-import org.apache.ignite.internal.causality.OutdatedTokenException;
-import org.apache.ignite.internal.causality.RevisionListenerRegistry;
-import org.apache.ignite.internal.causality.VersionedValue;
-import org.apache.ignite.internal.distributionzones.DataNodesMapSerializer;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.Augmentation;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.ZoneState;
-import org.apache.ignite.internal.distributionzones.DistributionZonesUtil;
-import org.apache.ignite.internal.distributionzones.Node;
-import org.apache.ignite.internal.distributionzones.NodeWithAttributes;
-import org.apache.ignite.internal.distributionzones.exception.DistributionZoneNotFoundException;
-import org.apache.ignite.internal.lang.ByteArray;
-import org.apache.ignite.internal.metastorage.Entry;
-import org.apache.ignite.internal.metastorage.MetaStorageManager;
-import org.apache.ignite.internal.util.IgniteSpinBusyLock;
-
-/**
- * Causality data nodes engine. Contains logic for obtaining zone's data nodes with causality token.
- */
-public class CausalityDataNodesEngine {
- /** Busy lock to stop synchronously. */
- private final IgniteSpinBusyLock busyLock;
-
- /** Meta Storage manager. */
- private final MetaStorageManager msManager;
-
- /** Distribution zones manager. */
- private final DistributionZoneManager distributionZoneManager;
-
- /** Catalog manager. */
- private final CatalogManager catalogManager;
-
- /**
- * Map with states for distribution zones. States are needed to track nodes that we want to add or remove from the data nodes,
- * schedule and stop scale up and scale down processes.
- */
- private final Map<Integer, ZoneState> zonesState;
-
- /**
- * The map which contains zones' create revision. It is updated only if the current node handled zone's create event.
- * TODO IGNITE-20050 Clean up this map.
- */
- private final ConcurrentHashMap<Integer, Long> zonesCreateRevision = new ConcurrentHashMap<>();
-
- /** Used to guarantee that the synchronous parts of all metastorage listeners for a specified causality token are handled. */
- private final VersionedValue<Void> zonesVv;
-
- /**
- * The constructor.
- *
- * @param busyLock Busy lock to stop synchronously.
- * @param registry Registry for versioned values.
- * @param msManager Meta Storage manager.
- * @param zonesState Map with states for distribution zones.
- * @param distributionZoneManager Distribution zones manager.
- * @param catalogManager Catalog manager.
- */
- public CausalityDataNodesEngine(
- IgniteSpinBusyLock busyLock,
- RevisionListenerRegistry registry,
- MetaStorageManager msManager,
- Map<Integer, ZoneState> zonesState,
- DistributionZoneManager distributionZoneManager,
- CatalogManager catalogManager
- ) {
- this.busyLock = busyLock;
- this.msManager = msManager;
- this.zonesState = zonesState;
- this.distributionZoneManager = distributionZoneManager;
- this.catalogManager = catalogManager;
-
- zonesVv = new IncrementalVersionedValue<>(registry);
- }
-
- /**
- * Gets data nodes of the zone using causality token and catalog version. {@code causalityToken} must be agreed
- * with the {@code catalogVersion}, meaning that for the provided {@code causalityToken} actual {@code catalogVersion} must be provided.
- * For example, if you are in the meta storage watch thread and {@code causalityToken} is the revision of the watch event, it is
- * safe to take {@link CatalogManager#latestCatalogVersion()} as a {@code catalogVersion},
- * because {@link CatalogManager#latestCatalogVersion()} won't be updated in a watch thread.
- * The same is applied for {@link CatalogEventParameters}, it is safe to take {@link CatalogEventParameters#causalityToken()}
- * as a {@code causalityToken} and {@link CatalogEventParameters#catalogVersion()} as a {@code catalogVersion}.
- *
- * <p>Return data nodes or throw the exception:
- * {@link IllegalArgumentException} if causalityToken or zoneId is not valid.
- * {@link DistributionZoneNotFoundException} if the zone with the provided zoneId does not exist.
- *
- * @param causalityToken Causality token.
- * @param catalogVersion Catalog version.
- * @param zoneId Zone id.
- * @return The future with data nodes for the zoneId.
- */
- public CompletableFuture<Set<String>> dataNodes(long causalityToken, int catalogVersion, int zoneId) {
- if (causalityToken < 1) {
- throw new IllegalArgumentException("causalityToken must be greater then zero [causalityToken=" + causalityToken + '"');
- }
-
- if (catalogVersion < 0) {
- throw new IllegalArgumentException("catalogVersion must be greater or equal to zero [catalogVersion=" + catalogVersion + '"');
- }
-
- if (zoneId < 0) {
- throw new IllegalArgumentException("zoneId cannot be a negative number [zoneId=" + zoneId + '"');
- }
-
- return inBusyLock(busyLock, () -> {
- try {
- return zonesVv.get(causalityToken);
- } catch (OutdatedTokenException e) {
- // This exception means that the DistributionZoneManager has already processed event with the causalityToken.
- return nullCompletedFuture();
- }
- }).thenApply(ignored -> inBusyLock(busyLock, () -> {
- CatalogZoneDescriptor zoneDescriptor = catalogManager.catalog(catalogVersion).zone(zoneId);
-
- if (zoneDescriptor == null) {
- // It means that the zone does not exist on a given causality token or causality token is lower than metastorage recovery
- // revision.
- throw new DistributionZoneNotFoundException(zoneId);
- }
-
- Long createRevision = zonesCreateRevision.get(zoneId);
-
- long descLastUpdateRevision = zoneDescriptor.updateToken();
-
- // Get revisions of the last scale up and scale down event which triggered immediate data nodes recalculation.
- long lastScaleUpRevision = getRevisionsOfLastScaleUpEvent(causalityToken, catalogVersion, zoneId);
- long lastScaleDownRevision = getRevisionsOfLastScaleDownEvent(causalityToken, catalogVersion, zoneId);
-
- Entry dataNodes = msManager.getLocally(zoneDataNodesKey(zoneId), causalityToken);
-
- if (createRevision != null && createRevision.equals(descLastUpdateRevision)
- && descLastUpdateRevision >= lastScaleUpRevision
- && descLastUpdateRevision >= lastScaleDownRevision
- && dataNodes.empty()
- ) {
- // It means that the zone was created but the data nodes value had not been updated yet.
- // So the data nodes value will be equals to the logical topology on the descLastUpdateRevision.
- Entry topologyEntry = msManager.getLocally(zonesLogicalTopologyKey(), createRevision);
-
- if (topologyEntry.empty()) {
- // A special case for a very first start of a node, when a creation of a zone was before the first topology event,
- // meaning that zonesLogicalTopologyKey() has not been updated yet, safe to return empty set here.
- return emptySet();
- }
-
- Set<NodeWithAttributes> logicalTopology = deserializeLogicalTopologySet(topologyEntry.value());
-
- Set<Node> logicalTopologyNodes = logicalTopology.stream().map(n -> n.node()).collect(toSet());
-
- return filterDataNodes(logicalTopologyNodes, zoneDescriptor, distributionZoneManager.nodesAttributes());
- }
-
- ZoneState zoneState = zonesState.get(zoneId);
-
- ConcurrentSkipListMap<Long, Augmentation> subAugmentationMap = null;
-
- // On the data nodes recalculation we write new data nodes to the meta storage then clear the augmentation map.
- // Therefore, first we need to read the augmentation map before it is cleared and then read the last data nodes value
- // from the meta storage.
- // The zoneState can be null if the zone was removed.
- if (zoneState != null) {
- subAugmentationMap = new ConcurrentSkipListMap<>(zoneState.topologyAugmentationMap()
- .headMap(causalityToken, true));
- }
-
- // Search the revisions of zoneScaleUpChangeTriggerKey and zoneScaleDownChangeTriggerKey with value greater or equals
- // to expected one.
- long scaleUpDataNodesRevision = searchTriggerKey(lastScaleUpRevision, zoneId, zoneScaleUpChangeTriggerKey(zoneId));
- long scaleDownDataNodesRevision = searchTriggerKey(lastScaleDownRevision, zoneId, zoneScaleDownChangeTriggerKey(zoneId));
-
- // Choose the highest revision.
- long dataNodesRevision = max(causalityToken, max(scaleUpDataNodesRevision, scaleDownDataNodesRevision));
-
- // Read data nodes value from the meta storage on dataNodesRevision and associated trigger keys.
- Entry dataNodesEntry = msManager.getLocally(zoneDataNodesKey(zoneId), dataNodesRevision);
- Entry scaleUpChangeTriggerKey = msManager.getLocally(zoneScaleUpChangeTriggerKey(zoneId), dataNodesRevision);
- Entry scaleDownChangeTriggerKey = msManager.getLocally(zoneScaleDownChangeTriggerKey(zoneId), dataNodesRevision);
-
- if (dataNodesEntry.value() == null) {
- // The zone was removed.
- // In this case it is impossible to find out the data nodes value idempotently.
- return emptySet();
- }
-
- Set<Node> baseDataNodes = DistributionZonesUtil.dataNodes(
- DataNodesMapSerializer.deserialize(dataNodesEntry.value())
- );
- long scaleUpTriggerRevision = bytesToLongKeepingOrder(scaleUpChangeTriggerKey.value());
- long scaleDownTriggerRevision = bytesToLongKeepingOrder(scaleDownChangeTriggerKey.value());
-
- Set<Node> finalDataNodes = new HashSet<>(baseDataNodes);
-
- // If the subAugmentationMap is null then it means that the zone was removed.
- // In this case all nodes from topologyAugmentationMap must be already written to the meta storage.
- if (subAugmentationMap != null) {
- // Update the data nodes set with pending data from augmentation map
- subAugmentationMap.forEach((rev, augmentation) -> {
- if (augmentation.addition() && rev > scaleUpTriggerRevision && rev <= lastScaleUpRevision) {
- finalDataNodes.addAll(augmentation.nodes());
- }
-
- if (!augmentation.addition() && rev > scaleDownTriggerRevision && rev <= lastScaleDownRevision) {
- finalDataNodes.removeAll(augmentation.nodes());
- }
- });
- }
-
- // Apply the filter to get the final data nodes set.
- return filterDataNodes(finalDataNodes, zoneDescriptor, distributionZoneManager.nodesAttributes());
- }));
- }
-
- /**
- * Return revision of the last event which triggers immediate scale up recalculation of the data nodes value.
- */
- private long getRevisionsOfLastScaleUpEvent(
- long causalityToken,
- int catalogVersion,
- int zoneId
- ) {
- return max(
- getLastScaleUpConfigRevision(catalogVersion, zoneId),
- getLastScaleUpTopologyRevisions(causalityToken, catalogVersion, zoneId)
- );
- }
-
- /**
- * Return revision of the last event which triggers immediate scale down recalculation of the data nodes value.
- */
- private long getRevisionsOfLastScaleDownEvent(
- long causalityToken,
- int catalogVersion,
- int zoneId
- ) {
- return max(
- getLastScaleDownConfigRevision(catalogVersion, zoneId),
- getLastScaleDownTopologyRevisions(causalityToken, catalogVersion, zoneId)
- );
- }
-
- /**
- * Return revision of the last configuration change event which triggers immediate scale up recalculation of the data nodes value.
- */
- private long getLastScaleUpConfigRevision(
- int catalogVersion,
- int zoneId
- ) {
- return getLastConfigRevision(catalogVersion, zoneId, true);
- }
-
- /**
- * Return revision of the last configuration change event which triggers immediate scale down recalculation of the data nodes value.
- */
- private long getLastScaleDownConfigRevision(
- int catalogVersion,
- int zoneId
- ) {
- return getLastConfigRevision(catalogVersion, zoneId, false);
- }
-
- /**
- * Get revision of the latest configuration change event which trigger immediate scale up or scale down recalculation
- * of the data nodes value.
- */
- private long getLastConfigRevision(
- int catalogVersion,
- int zoneId,
- boolean isScaleUp
- ) {
- CatalogZoneDescriptor entryNewerCfg = null;
-
- // Iterate over zone configurations from newest to oldest.
- for (int version = catalogVersion; version >= catalogManager.earliestCatalogVersion(); version--) {
- CatalogZoneDescriptor entryOlderCfg = catalogManager.catalog(version).zone(zoneId);
-
- if (entryOlderCfg == null) {
- break;
- }
-
- if (entryNewerCfg != null) {
- if (isScaleUp) {
- if (isScaleUpConfigRevision(entryOlderCfg, entryNewerCfg)) {
- return entryNewerCfg.updateToken();
- }
- } else {
- if (isScaleDownConfigRevision(entryOlderCfg, entryNewerCfg)) {
- return entryNewerCfg.updateToken();
- }
- }
- }
-
- entryNewerCfg = entryOlderCfg;
- }
-
- assert entryNewerCfg != null : "At least one zone configuration must be present .";
-
- // The case when there is only one configuration in the history. This configuration corresponds to the zone creation.
- return entryNewerCfg.updateToken();
- }
-
- /**
- * Check if newer configuration triggers immediate scale up recalculation of the data nodes value.
- * Return true if an older configuration has not immediate scale up and an newer configuration has immediate scale up
- * or older and newer configuration have different filter.
- */
- private static boolean isScaleUpConfigRevision(CatalogZoneDescriptor olderCfg, CatalogZoneDescriptor newerCfg) {
- return olderCfg.dataNodesAutoAdjustScaleUp() != newerCfg.dataNodesAutoAdjustScaleUp()
- && newerCfg.dataNodesAutoAdjustScaleUp() == IMMEDIATE_TIMER_VALUE
- || !olderCfg.filter().equals(newerCfg.filter());
- }
-
- /**
- * Check if newer configuration triggers immediate scale down recalculation of the data nodes value.
- * Return true if an older configuration has not immediate scale down and an newer configuration has immediate scale down.
- */
- private static boolean isScaleDownConfigRevision(CatalogZoneDescriptor olderCfg, CatalogZoneDescriptor newerCfg) {
- return olderCfg.dataNodesAutoAdjustScaleDown() != newerCfg.dataNodesAutoAdjustScaleDown()
- && newerCfg.dataNodesAutoAdjustScaleDown() == IMMEDIATE_TIMER_VALUE;
- }
-
- /**
- * Return revision of the last topology change event which triggers immediate scale up recalculation of the data nodes value.
- */
- private long getLastScaleUpTopologyRevisions(
- long causalityToken,
- int catalogVersion,
- int zoneId
- ) {
- return getLastTopologyRevisions(causalityToken, zoneId, catalogVersion, true);
- }
-
- /**
- * Return revision of the last topology change event which triggers immediate scale down recalculation of the data nodes value.
- */
- private long getLastScaleDownTopologyRevisions(
- long causalityToken,
- int catalogVersion,
- int zoneId
- ) {
- return getLastTopologyRevisions(causalityToken, zoneId, catalogVersion, false);
- }
-
- /**
- * Get revisions of the latest topology event with added nodes and with removed nodes when the zone have
- * immediate scale up and scale down timers.
- *
- * @param causalityToken causalityToken.
- * @param zoneId zoneId.
- * @return Revisions.
- */
- private long getLastTopologyRevisions(
- long causalityToken,
- int zoneId,
- int catalogVersion,
- boolean isScaleUp
- ) {
- Set<NodeWithAttributes> newerLogicalTopology;
-
- long newerTopologyRevision;
-
- Entry topologyEntry = msManager.getLocally(zonesLogicalTopologyKey(), causalityToken);
-
- if (!topologyEntry.empty()) {
- byte[] newerLogicalTopologyBytes = topologyEntry.value();
-
- newerLogicalTopology = deserializeLogicalTopologySet(newerLogicalTopologyBytes);
-
- newerTopologyRevision = topologyEntry.revision();
-
- while (true) {
- topologyEntry = msManager.getLocally(zonesLogicalTopologyKey(), newerTopologyRevision - 1);
-
- Set<NodeWithAttributes> olderLogicalTopology;
-
- if (topologyEntry.empty()) {
- // If older topology is empty then it means that each topology changes were iterated
- // so use empty set to compare it with the first topology.
- olderLogicalTopology = emptySet();
- } else {
- byte[] olderLogicalTopologyBytes = topologyEntry.value();
-
- olderLogicalTopology = deserializeLogicalTopologySet(olderLogicalTopologyBytes);
- }
-
- CatalogZoneDescriptor zoneDescriptor = catalogManager.catalog(catalogVersion).zone(zoneId);
-
- if (zoneDescriptor == null) {
- break;
- }
-
- if (isScaleUp) {
- if (isScaleUpTopologyRevision(olderLogicalTopology, newerLogicalTopology, zoneDescriptor)) {
- return newerTopologyRevision;
- }
- } else {
- if (isScaleDownTopologyRevision(olderLogicalTopology, newerLogicalTopology, zoneDescriptor)) {
- return newerTopologyRevision;
- }
- }
-
- newerLogicalTopology = olderLogicalTopology;
-
- newerTopologyRevision = topologyEntry.revision();
-
- if (topologyEntry.empty()) {
- break;
- }
- }
- }
-
- return 0;
- }
-
- /**
- * Check if newer topology triggers immediate scale up recalculation of the data nodes value.
- * Return true if newer logical topology has added nodes and a zone configuration has immediate scale up.
- */
- private static boolean isScaleUpTopologyRevision(
- Set<NodeWithAttributes> olderLogicalTopology,
- Set<NodeWithAttributes> newerLogicalTopology,
- CatalogZoneDescriptor zoneDescriptor
- ) {
- return newerLogicalTopology.stream().anyMatch(node -> !olderLogicalTopology.contains(node))
- && zoneDescriptor.dataNodesAutoAdjustScaleUp() == IMMEDIATE_TIMER_VALUE;
- }
-
- /**
- * Check if newer topology triggers immediate scale down recalculation of the data nodes value.
- * Return true if newer logical topology has removed nodes and a zone configuration has immediate scale down.
- */
- private static boolean isScaleDownTopologyRevision(
- Set<NodeWithAttributes> olderLogicalTopology,
- Set<NodeWithAttributes> newerLogicalTopology,
- CatalogZoneDescriptor zoneDescriptor
- ) {
- return olderLogicalTopology.stream().anyMatch(node -> !newerLogicalTopology.contains(node))
- && zoneDescriptor.dataNodesAutoAdjustScaleDown() == IMMEDIATE_TIMER_VALUE;
- }
-
- /**
- * Search a value of zoneScaleUpChangeTriggerKey/zoneScaleDownChangeTriggerKey which equals or greater than scaleRevision.
- * It iterates over the entries in the local meta storage. If there is an entry with a value equals to or greater than
- * the scaleRevision, then it returns the revision of this entry. If there is no such entry then it returns zero.
- *
- * @param scaleRevision Scale revision.
- * @param zoneId Zone id.
- * @param triggerKey Trigger key.
- * @return Revision.
- */
- private long searchTriggerKey(Long scaleRevision, int zoneId, ByteArray triggerKey) {
- Entry lastEntry = msManager.getLocally(triggerKey, Long.MAX_VALUE);
-
- long upperRevision = max(lastEntry.revision(), scaleRevision);
-
- // Gets old entries from storage to check if the expected value was handled before watch listener was registered.
- List<Entry> entryList = msManager.getLocally(triggerKey.bytes(), scaleRevision, upperRevision);
-
- for (Entry entry : entryList) {
- // scaleRevision is null if the zone was removed.
- if (entry.value() == null) {
- return entry.revision();
- } else {
- long entryScaleRevision = bytesToLongKeepingOrder(entry.value());
-
- if (entryScaleRevision >= scaleRevision) {
- return entry.revision();
- }
- }
- }
-
- return 0;
- }
-
- /**
- * Saves revision of the creation a zone to a local state.
- *
- * @param revision Revision.
- * @param zone Zone descriptor.
- */
- public void onCreateZoneState(long revision, CatalogZoneDescriptor zone) {
- int zoneId = zone.id();
-
- zonesCreateRevision.put(zoneId, revision);
- }
-
- /**
- * Update zone configuration on a zone dropping.
- *
- * @param revision Revision.
- * @param zoneId Zone id.
- */
- public void onDelete(long revision, int zoneId) {
- // https://issues.apache.org/jira/browse/IGNITE-20050
- }
-}
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngine.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngine.java
index 1275f8b7450..99b494cd24c 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngine.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngine.java
@@ -19,12 +19,13 @@ package org.apache.ignite.internal.distributionzones.rebalance;
import static java.util.concurrent.CompletableFuture.allOf;
import static java.util.concurrent.CompletableFuture.completedFuture;
+import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_ALTER;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.filterDataNodes;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.findTablesByZoneId;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.parseDataNodes;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryPrefix;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.extractZoneId;
import static org.apache.ignite.internal.lang.IgniteSystemProperties.enabledColocation;
import static org.apache.ignite.internal.lang.IgniteSystemProperties.getBoolean;
@@ -87,7 +88,7 @@ public class DistributionZoneRebalanceEngine {
private final DistributionZoneRebalanceEngineV2 distributionZoneRebalanceEngineV2;
/** Special flag to skip rebalance on node recovery for tests. */
- // TODO: IGNITE-23561 Remove it
+ // TODO: IGNITE-24607 Remove it
@TestOnly
public static final String SKIP_REBALANCE_TRIGGERS_RECOVERY = "IGNITE_SKIP_REBALANCE_TRIGGERS_RECOVERY";
@@ -132,7 +133,7 @@ public class DistributionZoneRebalanceEngine {
});
// TODO: IGNITE-18694 - Recovery for the case when zones watch listener processed event but assignments were not updated.
- metaStorageManager.registerPrefixWatch(zoneDataNodesKey(), dataNodesListener);
+ metaStorageManager.registerPrefixWatch(zoneDataNodesHistoryPrefix(), dataNodesListener);
CompletableFuture<Revisions> recoveryFinishFuture = metaStorageManager.recoveryFinishedFuture();
@@ -146,10 +147,10 @@ public class DistributionZoneRebalanceEngine {
}
if (enabledColocation()) {
- return rebalanceTriggersRecovery(recoveryRevision, catalogVersion)
+ return recoveryRebalanceTrigger(recoveryRevision, catalogVersion)
.thenCompose(v -> distributionZoneRebalanceEngineV2.startAsync());
} else {
- return rebalanceTriggersRecovery(recoveryRevision, catalogVersion);
+ return recoveryRebalanceTrigger(recoveryRevision, catalogVersion);
}
});
}
@@ -162,7 +163,7 @@ public class DistributionZoneRebalanceEngine {
// TODO: https://issues.apache.org/jira/browse/IGNITE-21058 At the moment this method produce many metastore multi-invokes
// TODO: which can be avoided by the local logic, which mirror the logic of metastore invokes.
// TODO: And then run the remote invoke, only if needed.
- private CompletableFuture<Void> rebalanceTriggersRecovery(long recoveryRevision, int catalogVersion) {
+ private CompletableFuture<Void> recoveryRebalanceTrigger(long recoveryRevision, int catalogVersion) {
if (recoveryRevision > 0) {
List<CompletableFuture<Void>> zonesRecoveryFutures = catalogService.catalog(catalogVersion).zones()
.stream()
@@ -198,14 +199,18 @@ public class DistributionZoneRebalanceEngine {
private WatchListener createDistributionZonesDataNodesListener() {
return evt -> IgniteUtils.inBusyLockAsync(busyLock, () -> {
- Set<Node> dataNodes = parseDataNodes(evt.entryEvent().newEntry().value());
+ Set<NodeWithAttributes> dataNodesWithAttributes = parseDataNodes(evt.entryEvent().newEntry().value(), evt.timestamp());
- if (dataNodes == null) {
+ if (dataNodesWithAttributes == null) {
// The zone was removed so data nodes was removed too.
return nullCompletedFuture();
}
- int zoneId = extractZoneId(evt.entryEvent().newEntry().key(), DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES);
+ Set<Node> dataNodes = dataNodesWithAttributes.stream()
+ .map(NodeWithAttributes::node)
+ .collect(toSet());
+
+ int zoneId = extractZoneId(evt.entryEvent().newEntry().key(), DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES);
// It is safe to get the latest version of the catalog as we are in the metastore thread.
// TODO: IGNITE-22723 Potentially unsafe to use the latest catalog version, as the tables might not already present
@@ -224,7 +229,9 @@ public class DistributionZoneRebalanceEngine {
Map<UUID, NodeWithAttributes> nodesAttributes = distributionZoneManager.nodesAttributes();
- Set<String> filteredDataNodes = filterDataNodes(dataNodes, zoneDescriptor, nodesAttributes);
+ Set<String> filteredDataNodes = filterDataNodes(dataNodesWithAttributes, zoneDescriptor).stream()
+ .map(NodeWithAttributes::nodeName)
+ .collect(toSet());
if (LOG.isInfoEnabled()) {
var matchedNodes = new ArrayList<NodeWithAttributes>();
@@ -323,7 +330,7 @@ public class DistributionZoneRebalanceEngine {
Set<String> aliveNodes = distributionZoneManager.logicalTopology(revision)
.stream()
.map(NodeWithAttributes::nodeName)
- .collect(Collectors.toSet());
+ .collect(toSet());
for (CatalogTableDescriptor tableDescriptor : tableDescriptors) {
tableFutures.add(RebalanceUtil.triggerAllTablePartitionsRebalance(
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineV2.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineV2.java
index 538fccafa52..a3a499afd71 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineV2.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineV2.java
@@ -19,11 +19,13 @@ package org.apache.ignite.internal.distributionzones.rebalance;
import static java.util.concurrent.CompletableFuture.allOf;
import static java.util.concurrent.CompletableFuture.completedFuture;
+import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_ALTER;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.filterDataNodes;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.parseDataNodes;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.extractZoneIdDataNodes;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryPrefix;
+import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.extractZoneId;
import static org.apache.ignite.internal.distributionzones.rebalance.ZoneRebalanceUtil.triggerZonePartitionsRebalance;
import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
@@ -114,7 +116,7 @@ public class DistributionZoneRebalanceEngineV2 {
});
// TODO: IGNITE-18694 - Recovery for the case when zones watch listener processed event but assignments were not updated.
- metaStorageManager.registerPrefixWatch(zoneDataNodesKey(), dataNodesListener);
+ metaStorageManager.registerPrefixWatch(zoneDataNodesHistoryPrefix(), dataNodesListener);
CompletableFuture<Revisions> recoveryFinishFuture = metaStorageManager.recoveryFinishedFuture();
@@ -123,7 +125,7 @@ public class DistributionZoneRebalanceEngineV2 {
long recoveryRevision = recoveryFinishFuture.join().revision();
- return rebalanceTriggersRecovery(recoveryRevision);
+ return recoveryRebalanceTrigger(recoveryRevision);
});
}
@@ -140,14 +142,18 @@ public class DistributionZoneRebalanceEngineV2 {
private WatchListener createDistributionZonesDataNodesListener() {
return evt -> IgniteUtils.inBusyLockAsync(busyLock, () -> {
- Set<Node> dataNodes = parseDataNodes(evt.entryEvent().newEntry().value());
+ Set<NodeWithAttributes> dataNodesWithAttributes = parseDataNodes(evt.entryEvent().newEntry().value(), evt.timestamp());
- if (dataNodes == null) {
+ if (dataNodesWithAttributes == null) {
// The zone was removed so data nodes were removed too.
return nullCompletedFuture();
}
- int zoneId = extractZoneIdDataNodes(evt.entryEvent().newEntry().key());
+ Set<Node> dataNodes = dataNodesWithAttributes.stream()
+ .map(NodeWithAttributes::node)
+ .collect(toSet());
+
+ int zoneId = extractZoneId(evt.entryEvent().newEntry().key(), DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES);
// It is safe to get the latest version of the catalog as we are in the metastore thread.
// TODO: IGNITE-22723 Potentially unsafe to use the latest catalog version, as the tables might not already present
@@ -167,7 +173,9 @@ public class DistributionZoneRebalanceEngineV2 {
Map<UUID, NodeWithAttributes> nodesAttributes = distributionZoneManager.nodesAttributes();
- Set<String> filteredDataNodes = filterDataNodes(dataNodes, zoneDescriptor, nodesAttributes);
+ Set<String> filteredDataNodes = filterDataNodes(dataNodesWithAttributes, zoneDescriptor).stream()
+ .map(NodeWithAttributes::nodeName)
+ .collect(toSet());
if (LOG.isInfoEnabled()) {
var matchedNodes = new ArrayList<NodeWithAttributes>();
@@ -205,7 +213,7 @@ public class DistributionZoneRebalanceEngineV2 {
Set<String> aliveNodes = distributionZoneManager.logicalTopology(revision)
.stream()
.map(NodeWithAttributes::nodeName)
- .collect(Collectors.toSet());
+ .collect(toSet());
return triggerZonePartitionsRebalance(
zoneDescriptor,
@@ -252,7 +260,7 @@ public class DistributionZoneRebalanceEngineV2 {
Set<String> aliveNodes = distributionZoneManager.logicalTopology(causalityToken)
.stream()
.map(NodeWithAttributes::nodeName)
- .collect(Collectors.toSet());
+ .collect(toSet());
return triggerZonePartitionsRebalance(
zoneDescriptor,
@@ -274,7 +282,7 @@ public class DistributionZoneRebalanceEngineV2 {
// TODO: https://issues.apache.org/jira/browse/IGNITE-21058 At the moment this method produce many metastore multi-invokes
// TODO: which can be avoided by the local logic, which mirror the logic of metastore invokes.
// TODO: And then run the remote invoke, only if needed.
- private CompletableFuture<Void> rebalanceTriggersRecovery(long recoveryRevision) {
+ private CompletableFuture<Void> recoveryRebalanceTrigger(long recoveryRevision) {
if (recoveryRevision > 0) {
Catalog catalog = catalogService.catalog(catalogService.latestCatalogVersion());
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
index 129c58ce5a0..9c81eb1e650 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/RebalanceUtil.java
@@ -172,8 +172,9 @@ public class RebalanceUtil {
// All complicated logic here is needed because we want to return back to stable nodes
// that are returned back after majority is lost and stable was narrowed.
// Let's consider example:
+ // scale down big enough (for example, infinite)
// stable = [A, B, C], dataNodes = [A, B, C]
- // B, C left, stable = [A], dataNodes = [A, B, C]
+ // B, C left, stable = [A] due to partition reset, dataNodes = [A, B, C]
// B returned, we want stable = [A, B], but in terms of data nodes they are not changed and equal [A, B, C]
// So, because scale up mechanism in this case won't adjust stable, we need to add B to stable manually.
// General idea is to filter offline nodes from data nodes, but we need to be careful and do not remove nodes
diff --git a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
index d702d1931a0..54be01f8f7c 100644
--- a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
+++ b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/rebalance/ZoneRebalanceUtil.java
@@ -21,7 +21,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.CompletableFuture.allOf;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.UpdateStatus.ASSIGNMENT_NOT_UPDATED;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.UpdateStatus.OUTDATED_UPDATE_RECEIVED;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.UpdateStatus.PENDING_KEY_UPDATED;
@@ -54,7 +53,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.IntStream;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
-import org.apache.ignite.internal.distributionzones.DistributionZonesUtil;
import org.apache.ignite.internal.lang.ByteArray;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
@@ -504,27 +502,6 @@ public class ZoneRebalanceUtil {
return ZonePartitionId.fromString(zonePartitionIdString);
}
- /**
- * Extract zone id from a metastorage key {@link DistributionZonesUtil#zoneDataNodesKey()}.
- *
- * @param key Key.
- * @return Table id.
- */
- public static int extractZoneIdDataNodes(byte[] key) {
- return Integer.parseInt(toStringWithoutPrefix(key, DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES.length));
- }
-
- /**
- * Checks if an error is recoverable, so we can retry a rebalance intent.
- *
- * @param t The throwable.
- * @return {@code True} if this is a recoverable exception.
- */
- public static boolean recoverable(Throwable t) {
- // As long as we don't have a general failure handler, we assume that all errors are recoverable.
- return true;
- }
-
/**
* Removes nodes from set of nodes.
*
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/BaseDistributionZoneManagerTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/BaseDistributionZoneManagerTest.java
index 4cb6aa5bed5..092bdde344b 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/BaseDistributionZoneManagerTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/BaseDistributionZoneManagerTest.java
@@ -22,20 +22,20 @@ import static java.util.concurrent.CompletableFuture.completedFuture;
import static org.apache.ignite.internal.catalog.CatalogTestUtils.createTestCatalogManager;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.PARTITION_DISTRIBUTION_RESET_TIMEOUT;
-import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.REBALANCE_SCHEDULER_POOL_SIZE;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
import static org.apache.ignite.internal.util.IgniteUtils.closeAll;
import static org.apache.ignite.internal.util.IgniteUtils.startAsync;
import static org.apache.ignite.internal.util.IgniteUtils.stopAsync;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
import org.apache.ignite.internal.catalog.CatalogManager;
import org.apache.ignite.internal.catalog.CatalogTestUtils;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
@@ -49,8 +49,11 @@ import org.apache.ignite.internal.cluster.management.topology.LogicalTopologySer
import org.apache.ignite.internal.configuration.SystemDistributedConfiguration;
import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
import org.apache.ignite.internal.configuration.testframework.InjectConfiguration;
+import org.apache.ignite.internal.hlc.ClockWaiter;
import org.apache.ignite.internal.hlc.HybridClock;
import org.apache.ignite.internal.hlc.HybridClockImpl;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.hlc.TestClockService;
import org.apache.ignite.internal.manager.ComponentContext;
import org.apache.ignite.internal.manager.IgniteComponent;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
@@ -84,7 +87,7 @@ public abstract class BaseDistributionZoneManagerTest extends BaseIgniteAbstract
protected MetaStorageManager metaStorageManager;
- private final HybridClock clock = new HybridClockImpl();
+ protected final HybridClock clock = new HybridClockImpl();
protected CatalogManager catalogManager;
@@ -120,8 +123,9 @@ public abstract class BaseDistributionZoneManagerTest extends BaseIgniteAbstract
catalogManager = createTestCatalogManager(nodeName, clock, metaStorageManager);
components.add(catalogManager);
- ScheduledExecutorService rebalanceScheduler = new ScheduledThreadPoolExecutor(REBALANCE_SCHEDULER_POOL_SIZE,
- NamedThreadFactory.create(nodeName, "test-rebalance-scheduler", logger()));
+ ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
+ NamedThreadFactory.create(nodeName, "distribution-zone-manager-test-scheduled-executor", log)
+ );
distributionZoneManager = new DistributionZoneManager(
nodeName,
@@ -129,8 +133,8 @@ public abstract class BaseDistributionZoneManagerTest extends BaseIgniteAbstract
metaStorageManager,
new LogicalTopologyServiceImpl(topology, cmgManager),
catalogManager,
- rebalanceScheduler,
- systemDistributedConfiguration
+ systemDistributedConfiguration,
+ new TestClockService(clock, new ClockWaiter(nodeName, clock, scheduledExecutorService))
);
// Not adding 'distributionZoneManager' on purpose, it's started manually.
@@ -223,9 +227,15 @@ public abstract class BaseDistributionZoneManagerTest extends BaseIgniteAbstract
return DistributionZonesTestUtil.getZoneIdStrict(catalogManager, zoneName, clock.nowLong());
}
- protected CatalogZoneDescriptor getDefaultZone() {
- CatalogTestUtils.awaitDefaultZoneCreation(catalogManager);
+ protected CatalogZoneDescriptor zoneDescriptor(String zoneName, HybridTimestamp timestamp) {
+ CatalogZoneDescriptor zoneDescriptor = catalogManager.activeCatalog(timestamp.longValue()).zone(zoneName);
+
+ assertNotNull(zoneDescriptor);
- return DistributionZonesTestUtil.getDefaultZone(catalogManager, clock.nowLong());
+ return zoneDescriptor;
+ }
+
+ protected CatalogZoneDescriptor getDefaultZone() {
+ return CatalogTestUtils.awaitDefaultZoneCreation(catalogManager);
}
}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesHistorySerializerTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesHistorySerializerTest.java
new file mode 100644
index 00000000000..10ddaedce18
--- /dev/null
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesHistorySerializerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link DataNodesHistorySerializer}.
+ */
+public class DataNodesHistorySerializerTest {
+ private static final UUID NODE_A_ID = new UUID(0x1234567890ABCDEFL, 0xFEDCBA0987654321L);
+ private static final UUID NODE_B_ID = new UUID(0xFEDCBA0987654321L, 0x1234567890ABCDEFL);
+
+ private static final NodeWithAttributes A = new NodeWithAttributes("node_A", NODE_A_ID, emptyMap(), emptyList());
+ private static final NodeWithAttributes B = new NodeWithAttributes(
+ "node_B",
+ NODE_B_ID,
+ Map.of("region", "US"),
+ List.of("default")
+ );
+
+ private static final HybridTimestamp TIMESTAMP_0 = new HybridTimestamp(1234, 10);
+ private static final HybridTimestamp TIMESTAMP_1 = new HybridTimestamp(12341234, 20);
+
+ private static final String SERIALIZED_V1 = "Ae++QwMKANIEAAAAAAIB775DAe++Qwdub2RlX0HvzauQeFY0EiFDZYcJutz+AQEUAPJPvAAAAAMB775DAe++Qwd"
+ + "ub2RlX0HvzauQeFY0EiFDZYcJutz+AQEB775DAe++Qwdub2RlX0IhQ2WHCbrc/u/Nq5B4VjQSAgdyZWdpb24DVVMCCGRlZmF1bHQ=";
+
+ @Test
+ public void serializationAndDeserializationOfEmpty() {
+ DataNodesHistory history = new DataNodesHistory();
+ byte[] bytes = DataNodesHistorySerializer.serialize(history);
+ DataNodesHistory deserializedHistory = DataNodesHistorySerializer.deserialize(bytes);
+ assertEquals(history, deserializedHistory);
+ assertTrue(history.isEmpty());
+ }
+
+ @Test
+ public void serializationAndDeserialization() {
+ DataNodesHistory history = new DataNodesHistory();
+ history = history.addHistoryEntry(TIMESTAMP_0, Set.of(A));
+ history = history.addHistoryEntry(TIMESTAMP_1, Set.of(A, B));
+
+ byte[] bytes = DataNodesHistorySerializer.serialize(history);
+ DataNodesHistory deserializedHistory = DataNodesHistorySerializer.deserialize(bytes);
+ assertEquals(history, deserializedHistory);
+
+ assertTrue(deserializedHistory.entryIsPresentAtExactTimestamp(TIMESTAMP_0));
+ assertTrue(deserializedHistory.entryIsPresentAtExactTimestamp(TIMESTAMP_1));
+
+ assertEquals(new DataNodesHistoryEntry(TIMESTAMP_0, Set.of(A)), deserializedHistory.dataNodesForTimestamp(TIMESTAMP_0));
+ assertEquals(new DataNodesHistoryEntry(TIMESTAMP_1, Set.of(A, B)), deserializedHistory.dataNodesForTimestamp(TIMESTAMP_1));
+ }
+
+ @Test
+ public void v1CanBeDeserialized() {
+ byte[] serializedBytes = Base64.getDecoder().decode(SERIALIZED_V1);
+ DataNodesHistory deserializedHistory = DataNodesHistorySerializer.deserialize(serializedBytes);
+ assertTrue(deserializedHistory.entryIsPresentAtExactTimestamp(TIMESTAMP_0));
+ assertTrue(deserializedHistory.entryIsPresentAtExactTimestamp(TIMESTAMP_1));
+
+ assertEquals(new DataNodesHistoryEntry(TIMESTAMP_0, Set.of(A)), deserializedHistory.dataNodesForTimestamp(TIMESTAMP_0));
+ assertEquals(new DataNodesHistoryEntry(TIMESTAMP_1, Set.of(A, B)), deserializedHistory.dataNodesForTimestamp(TIMESTAMP_1));
+ }
+}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesManagerTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesManagerTest.java
new file mode 100644
index 00000000000..3fc484df7d7
--- /dev/null
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesManagerTest.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+import static org.apache.ignite.internal.catalog.CatalogTestUtils.createTestCatalogManager;
+import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
+import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
+import static org.apache.ignite.internal.catalog.descriptors.ConsistencyMode.HIGH_AVAILABILITY;
+import static org.apache.ignite.internal.catalog.descriptors.ConsistencyMode.STRONG_CONSISTENCY;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
+import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
+import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast;
+import static org.apache.ignite.internal.util.ArrayUtils.asList;
+import static org.apache.ignite.internal.util.CompletableFutures.allOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+import org.apache.ignite.internal.catalog.CatalogManager;
+import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
+import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.distributionzones.DataNodesManager.ZoneTimerSchedule;
+import org.apache.ignite.internal.hlc.ClockService;
+import org.apache.ignite.internal.hlc.ClockWaiter;
+import org.apache.ignite.internal.hlc.HybridClock;
+import org.apache.ignite.internal.hlc.HybridClockImpl;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.hlc.TestClockService;
+import org.apache.ignite.internal.manager.ComponentContext;
+import org.apache.ignite.internal.metastorage.Entry;
+import org.apache.ignite.internal.metastorage.MetaStorageManager;
+import org.apache.ignite.internal.metastorage.impl.StandaloneMetaStorageManager;
+import org.apache.ignite.internal.metastorage.server.KeyValueStorage;
+import org.apache.ignite.internal.metastorage.server.ReadOperationForCompactionTracker;
+import org.apache.ignite.internal.metastorage.server.SimpleInMemoryKeyValueStorage;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.internal.thread.NamedThreadFactory;
+import org.apache.ignite.internal.util.IgniteSpinBusyLock;
+import org.jetbrains.annotations.Nullable;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link DataNodesManager}.
+ */
+public class DataNodesManagerTest extends BaseIgniteAbstractTest {
+ private static final String ZONE_NAME_1 = "test_zone_1";
+ private static final String ZONE_NAME_2 = "test_zone_2";
+
+ private static final NodeWithAttributes A = new NodeWithAttributes("A", UUID.randomUUID(), emptyMap(), List.of("default"));
+ private static final NodeWithAttributes B = new NodeWithAttributes("B", UUID.randomUUID(), emptyMap(), List.of("default"));
+
+ private static final NodeWithAttributes C = new NodeWithAttributes(
+ "C",
+ UUID.randomUUID(),
+ Map.of("region", "US"),
+ List.of("default")
+ );
+
+ private static final NodeWithAttributes C_DIFFERENT_ATTRS = new NodeWithAttributes(
+ "C",
+ UUID.randomUUID(),
+ Map.of("region", "EU"),
+ List.of("default")
+ );
+
+ private static final NodeWithAttributes D = new NodeWithAttributes("D", UUID.randomUUID(), emptyMap(), List.of("default"));
+
+ private static final String NODE_NAME = "node";
+
+ private KeyValueStorage storage;
+ private HybridClock clock;
+ private MetaStorageManager metaStorageManager;
+ private CatalogManager catalogManager;
+ private DataNodesManager dataNodesManager;
+
+ private Set<NodeWithAttributes> currentTopology;
+
+ private final AtomicBoolean partitionResetTriggered = new AtomicBoolean();
+
+ private final BiConsumer<Long, Integer> partitionResetClosure = (revision, zoneId) -> partitionResetTriggered.set(true);
+
+ @BeforeEach
+ public void setUp() {
+ ComponentContext startComponentContext = new ComponentContext();
+
+ ReadOperationForCompactionTracker readOperationForCompactionTracker = new ReadOperationForCompactionTracker();
+ storage = new SimpleInMemoryKeyValueStorage(NODE_NAME, readOperationForCompactionTracker);
+ clock = new HybridClockImpl();
+
+ metaStorageManager = StandaloneMetaStorageManager.create(storage, readOperationForCompactionTracker);
+ assertThat(metaStorageManager.startAsync(startComponentContext), willCompleteSuccessfully());
+ assertThat(metaStorageManager.recoveryFinishedFuture(), willCompleteSuccessfully());
+
+ catalogManager = createTestCatalogManager(NODE_NAME, clock, metaStorageManager);
+
+ ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
+ NamedThreadFactory.create(NODE_NAME, "data-nodes-manager-test-scheduled-executor", log)
+ );
+
+ ClockService clockService = new TestClockService(clock, new ClockWaiter(NODE_NAME, clock, scheduledExecutorService));
+
+ dataNodesManager = new DataNodesManager(
+ NODE_NAME,
+ new IgniteSpinBusyLock(),
+ metaStorageManager,
+ catalogManager,
+ clockService,
+ partitionResetClosure,
+ () -> 1
+ );
+
+ currentTopology = new HashSet<>(Set.of(A, B));
+
+ assertThat(catalogManager.startAsync(startComponentContext), willCompleteSuccessfully());
+ assertThat(dataNodesManager.startAsync(emptyList(), 1), willCompleteSuccessfully());
+
+ metaStorageManager.deployWatches();
+
+ createZone(ZONE_NAME_1, STRONG_CONSISTENCY);
+ createZone(ZONE_NAME_2, HIGH_AVAILABILITY);
+
+ dataNodesManager.onZoneCreate(0, clock.now(), currentTopology);
+ }
+
+ private void createZone(String name, ConsistencyMode consistencyMode) {
+ DistributionZonesTestUtil.createZone(catalogManager, name, consistencyMode);
+
+ assertThat(dataNodesManager.onZoneCreate(zoneId(name), clock.now(), currentTopology), willCompleteSuccessfully());
+ }
+
+ private void alterZone(
+ String zoneName,
+ @Nullable Integer dataNodesAutoAdjustScaleUp,
+ @Nullable Integer dataNodesAutoAdjustScaleDown,
+ @Nullable String filter
+ ) {
+ CatalogZoneDescriptor oldDescriptor = descriptor(zoneName);
+
+ DistributionZonesTestUtil.alterZone(catalogManager, zoneName, dataNodesAutoAdjustScaleUp, dataNodesAutoAdjustScaleDown, filter);
+
+ if (dataNodesAutoAdjustScaleUp != null || dataNodesAutoAdjustScaleDown != null) {
+ dataNodesManager.onAutoAdjustAlteration(
+ descriptor(zoneName),
+ clock.now(),
+ oldDescriptor.dataNodesAutoAdjustScaleUp(),
+ oldDescriptor.dataNodesAutoAdjustScaleDown()
+ );
+ }
+
+ if (filter != null) {
+ dataNodesManager.onZoneFilterChange(descriptor(ZONE_NAME_1), clock.now(), currentTopology);
+ }
+ }
+
+ @Test
+ public void addNodesScaleUpImmediate() throws InterruptedException {
+ addNodes(Set.of(C));
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+ }
+
+ @Test
+ public void addMultipleNodesScaleUpImmediate() throws InterruptedException {
+ addNodes(Set.of(C, D));
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C, D));
+ }
+
+ @Test
+ public void addNodesScaleUpChangedToInfinite() throws InterruptedException {
+ alterZone(ZONE_NAME_1, INFINITE_TIMER_VALUE, null, null);
+
+ addNodes(Set.of(C));
+
+ assertScaleUpNotScheduled(ZONE_NAME_1);
+ checkDataNodes(ZONE_NAME_1, clock.now(), nodeNames(A, B));
+
+ // Other zone was not altered and should change its data nodes.
+ waitForDataNodes(ZONE_NAME_2, nodeNames(A, B, C));
+ }
+
+ @Test
+ public void addNodesWithScheduledScaleUp() throws InterruptedException {
+ alterZone(ZONE_NAME_1, 1, null, null);
+
+ addNodes(Set.of(C));
+
+ // Task is scheduled after C added.
+ assertScaleUpScheduledOrDone(ZONE_NAME_1);
+ assertScaleDownNotScheduled(ZONE_NAME_1);
+
+ addNodes(Set.of(D));
+
+ // Check that scale up tasks will be merged and both nodes will be finally added.
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C, D));
+ }
+
+ @Test
+ public void removeNodesScaleDown() throws InterruptedException {
+ removeNodes(Set.of(B));
+
+ assertScaleDownNotScheduled(ZONE_NAME_1);
+ checkDataNodes(ZONE_NAME_1, clock.now(), nodeNames(A, B));
+ }
+
+ @Test
+ public void removeMultipleNodesScaleDownImmediate() throws InterruptedException {
+ alterZone(ZONE_NAME_1, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, null);
+
+ removeNodes(Set.of(A, B));
+
+ waitForDataNodes(ZONE_NAME_1, Set.of());
+ }
+
+ @Test
+ public void removeNodesScaleDownChangedToImmediate() throws InterruptedException {
+ alterZone(ZONE_NAME_1, null, IMMEDIATE_TIMER_VALUE, null);
+
+ removeNodes(Set.of(B));
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A));
+
+ // Other zone was not altered and should change its data nodes.
+ assertScaleDownNotScheduled(ZONE_NAME_2);
+ checkDataNodes(ZONE_NAME_2, clock.now(), nodeNames(A, B));
+ }
+
+ @Test
+ public void removeNodesWithScheduledScaleDown() throws InterruptedException {
+ alterZone(ZONE_NAME_1, null, 1, null);
+
+ removeNodes(Set.of(A));
+
+ // Task is scheduled after A removed.
+ assertScaleUpNotScheduled(ZONE_NAME_1);
+ assertScaleDownScheduledOrDone(ZONE_NAME_1);
+
+ removeNodes(Set.of(B));
+
+ // Check that scale down tasks will be merged and both nodes will be finally removed.
+ waitForDataNodes(ZONE_NAME_1, Set.of());
+ }
+
+ @Test
+ public void previousDataNodesAreAvailable() throws InterruptedException {
+ HybridTimestamp time = clock.now();
+
+ addNodes(Set.of(C));
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+
+ CompletableFuture<Set<String>> previousDataNodes = dataNodesManager.dataNodes(zoneId(ZONE_NAME_1), time);
+ assertThat(previousDataNodes, willCompleteSuccessfully());
+ assertEquals(nodeNames(A, B), previousDataNodes.join());
+ }
+
+ @Test
+ public void filterChange() throws InterruptedException {
+ alterZone(ZONE_NAME_1, 10, 10, null);
+
+ addNodes(Set.of(C));
+
+ assertScaleUpScheduledOrDone(ZONE_NAME_1);
+
+ removeNodes(Set.of(A));
+
+ assertScaleDownScheduledOrDone(ZONE_NAME_1);
+
+ String newFilter = "$[?(@.region == \"US\")]";
+
+ alterZone(ZONE_NAME_1, null, null, newFilter);
+
+ checkDataNodes(ZONE_NAME_1, clock.now(), nodeNames(C));
+
+ // Timers are discarded earlier than scheduled.
+ assertTrue(waitForCondition(() -> !scaleUpScheduled(ZONE_NAME_1) && !scaleDownScheduled(ZONE_NAME_1), 2000));
+ }
+
+ @Test
+ public void scaleUpChange() throws InterruptedException {
+ // Scale up time is big enough.
+ alterZone(ZONE_NAME_1, 100, null, null);
+
+ addNodes(Set.of(C));
+
+ assertScaleUpScheduledOrDone(ZONE_NAME_1);
+
+ alterZone(ZONE_NAME_1, IMMEDIATE_TIMER_VALUE, null, null);
+
+ // After the change, scheduled scale up is applied.
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+ }
+
+ @Test
+ public void scaleDownChange() throws InterruptedException {
+ // Scale down time is big enough.
+ alterZone(ZONE_NAME_1, null, 100, null);
+
+ removeNodes(Set.of(A));
+
+ assertScaleDownScheduledOrDone(ZONE_NAME_1);
+
+ alterZone(ZONE_NAME_1, null, IMMEDIATE_TIMER_VALUE, null);
+
+ // After the change, scheduled scale up is applied.
+ waitForDataNodes(ZONE_NAME_1, nodeNames(B));
+ }
+
+ @Test
+ public void dataNodesManagerRecovery() throws InterruptedException {
+ alterZone(ZONE_NAME_1, 100, 100, null);
+
+ addNodes(Set.of(C));
+ removeNodes(Set.of(A));
+
+ assertScaleUpScheduledOrDone(ZONE_NAME_1);
+ assertScaleDownScheduledOrDone(ZONE_NAME_1);
+
+ dataNodesManager.stop();
+
+ assertScaleUpNotScheduled(ZONE_NAME_1);
+ assertScaleDownNotScheduled(ZONE_NAME_1);
+
+ assertThat(
+ dataNodesManager.startAsync(catalogManager.activeCatalog(clock.now().longValue()).zones(), 1), willCompleteSuccessfully()
+ );
+
+ assertScaleUpScheduledOrDone(ZONE_NAME_1);
+ assertScaleDownScheduledOrDone(ZONE_NAME_1);
+ }
+
+ @Test
+ public void partitionResetTriggeredOnNodeRemove() throws InterruptedException {
+ partitionResetTriggered.set(false);
+
+ CatalogZoneDescriptor zone = descriptor(ZONE_NAME_2);
+
+ dataNodesManager.onTopologyChange(zone, 1, clock.now(), currentTopology, currentTopology);
+
+ // Partition reset is not triggered if no nodes were removed.
+ assertFalse(partitionResetTriggered.get());
+
+ Set<NodeWithAttributes> newTopology = new HashSet<>(currentTopology);
+ newTopology.remove(A);
+
+ dataNodesManager.onTopologyChange(zone, 1, clock.now(), newTopology, currentTopology);
+
+ assertTrue(waitForCondition(partitionResetTriggered::get, 2000));
+ }
+
+ @Test
+ public void nodeRejoin() throws InterruptedException {
+ removeNodes(Set.of(A));
+ addNodes(Set.of(A));
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B));
+ }
+
+ @Test
+ public void nodeRejoinWithAnotherNodeJoin() throws InterruptedException {
+ addNodes(Set.of(C));
+ removeNodes(Set.of(A));
+ addNodes(Set.of(A));
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+ }
+
+ @Test
+ public void nodeRejoinWithAnotherNodeJoinWithAutoAdjustChange() throws InterruptedException {
+ alterZone(ZONE_NAME_1, 100, 100, null);
+
+ addNodes(Set.of(C));
+ removeNodes(Set.of(A));
+ addNodes(Set.of(A));
+
+ alterZone(ZONE_NAME_1, 0, 0, null);
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+ }
+
+ @Test
+ public void nodeRejoinWithDifferentAttributes() throws InterruptedException {
+ removeNodes(Set.of(C));
+ addNodes(Set.of(C_DIFFERENT_ATTRS));
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+
+ NodeWithAttributes c = nodeFromHistory(dataNodesHistory(ZONE_NAME_1), C.nodeName(), HybridTimestamp.MAX_VALUE);
+
+ assertEquals(C_DIFFERENT_ATTRS.userAttributes().get("region"), c.userAttributes().get("region"));
+ }
+
+ @Test
+ public void nodeRejoinWithDifferentAttributesWithAutoAdjustAlteration() throws InterruptedException {
+ alterZone(ZONE_NAME_1, 100, 100, null);
+
+ removeNodes(Set.of(C));
+ addNodes(Set.of(C_DIFFERENT_ATTRS));
+
+ alterZone(ZONE_NAME_1, 0, 0, null);
+
+ waitForDataNodes(ZONE_NAME_1, nodeNames(A, B, C));
+
+ NodeWithAttributes c = nodeFromHistory(dataNodesHistory(ZONE_NAME_1), C.nodeName(), HybridTimestamp.MAX_VALUE);
+
+ assertEquals(C_DIFFERENT_ATTRS.userAttributes().get("region"), c.userAttributes().get("region"));
+ }
+
+ private static NodeWithAttributes nodeFromHistory(DataNodesHistory history, String nodeName, HybridTimestamp timestamp) {
+ return history.dataNodesForTimestamp(timestamp).dataNodes().stream()
+ .filter(n -> n.nodeName().equals(nodeName))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ private DataNodesHistory dataNodesHistory(String zoneName) {
+ CompletableFuture<Entry> entryFut = metaStorageManager.get(zoneDataNodesHistoryKey(zoneId(zoneName)));
+ assertThat(entryFut, willCompleteSuccessfully());
+ Entry e = entryFut.join();
+
+ assertNotNull(e);
+ assertNotNull(e.value());
+
+ return DataNodesHistorySerializer.deserialize(e.value());
+ }
+
+ private void addNodes(Set<NodeWithAttributes> nodes) {
+ currentTopology.addAll(nodes);
+
+ assertThat(
+ allOf(
+ catalogManager.activeCatalog(clock.now().longValue()).zones()
+ .stream()
+ .map(zone -> dataNodesManager
+ .onTopologyChange(zone, 1, clock.now(), currentTopology, currentTopology))
+ .collect(toList())
+ ),
+ willCompleteSuccessfully()
+ );
+ }
+
+ private void removeNodes(Set<NodeWithAttributes> nodes) {
+ currentTopology.removeAll(nodes);
+
+ assertThat(
+ allOf(
+ catalogManager.activeCatalog(clock.now().longValue()).zones()
+ .stream()
+ .map(zone -> dataNodesManager
+ .onTopologyChange(zone, 1, clock.now(), currentTopology, currentTopology))
+ .collect(toList())
+ ),
+ willCompleteSuccessfully()
+ );
+ }
+
+ private void waitForDataNodes(String zoneName, Set<String> expectedNodes) throws InterruptedException {
+ CatalogZoneDescriptor zone = catalogManager.activeCatalog(clock.now().longValue()).zone(zoneName);
+ int zoneId = zone.id();
+
+ boolean success = waitForCondition(() -> {
+ CompletableFuture<Set<String>> dataNodesFuture = dataNodesManager.dataNodes(zoneId, clock.now());
+ assertThat(dataNodesFuture, willSucceedFast());
+ return dataNodesFuture.join().equals(expectedNodes);
+ }, 5000);
+
+ if (!success) {
+ System.out.println("Expected: " + expectedNodes);
+ System.out.println("Actual: " + dataNodesManager.dataNodes(zoneId, clock.now()).join());
+ }
+
+ assertTrue(success);
+ }
+
+ private void checkDataNodes(String zoneName, HybridTimestamp timestamp, Set<String> expectedNodes) {
+ CatalogZoneDescriptor zone = catalogManager.activeCatalog(timestamp.longValue()).zone(zoneName);
+ int zoneId = zone.id();
+
+ CompletableFuture<Set<String>> dataNodesFuture = dataNodesManager.dataNodes(zoneId, clock.now());
+ assertThat(dataNodesFuture, willSucceedFast());
+ assertEquals(expectedNodes, dataNodesFuture.join());
+ }
+
+ private CatalogZoneDescriptor descriptor(String zoneName) {
+ CatalogZoneDescriptor zoneDescriptor = catalogManager.activeCatalog(clock.now().longValue()).zone(zoneName);
+ assertNotNull(zoneDescriptor);
+ return zoneDescriptor;
+ }
+
+ private int zoneId(String zoneName) {
+ return descriptor(zoneName).id();
+ }
+
+ private void assertScaleUpScheduledOrDone(String zoneName) throws InterruptedException {
+ assertTrue(waitForCondition(() -> {
+ ZoneTimerSchedule schedule = dataNodesManager.zoneTimers(zoneId(zoneName)).scaleUp;
+ return schedule.taskIsScheduled() || schedule.taskIsDone();
+ }, 2000));
+ }
+
+ private void assertScaleUpNotScheduled(String zoneName) throws InterruptedException {
+ assertFalse(waitForCondition(() -> dataNodesManager.zoneTimers(zoneId(zoneName)).scaleUp.taskIsScheduled(), 1000));
+ }
+
+ private void assertScaleDownScheduledOrDone(String zoneName) throws InterruptedException {
+ boolean success = waitForCondition(() -> {
+ ZoneTimerSchedule schedule = dataNodesManager.zoneTimers(zoneId(zoneName)).scaleDown;
+ return schedule.taskIsScheduled() || schedule.taskIsDone();
+ }, 2000);
+
+ if (!success) {
+ ZoneTimerSchedule schedule = dataNodesManager.zoneTimers(zoneId(zoneName)).scaleDown;
+ System.out.println(schedule);
+ }
+
+ assertTrue(success);
+ }
+
+ private void assertScaleDownNotScheduled(String zoneName) throws InterruptedException {
+ assertFalse(waitForCondition(() -> dataNodesManager.zoneTimers(zoneId(zoneName)).scaleDown.taskIsScheduled(), 1000));
+ }
+
+ private boolean scaleUpScheduled(String zoneName) {
+ return dataNodesManager.zoneTimers(zoneId(zoneName)).scaleUp.taskIsScheduled();
+ }
+
+ private boolean scaleDownScheduled(String zoneName) {
+ return dataNodesManager.zoneTimers(zoneId(zoneName)).scaleDown.taskIsScheduled();
+ }
+
+ private static Set<String> nodeNames(NodeWithAttributes... nodes) {
+ return asList(nodes).stream().map(NodeWithAttributes::nodeName).collect(toSet());
+ }
+}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesMapSerializerTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesMapSerializerTest.java
deleted file mode 100644
index 5edd1c17544..00000000000
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DataNodesMapSerializerTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.distributionzones;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.aMapWithSize;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.is;
-
-import java.util.Base64;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.internal.versioned.VersionedSerialization;
-import org.junit.jupiter.api.Test;
-
-class DataNodesMapSerializerTest {
- private final DataNodesMapSerializer serializer = new DataNodesMapSerializer();
-
- @Test
- void serializationAndDeserialization() {
- Map<Node, Integer> originalMap = Map.of(
- new Node("a", new UUID(0x1234567890ABCDEFL, 0xFEDCBA0987654321L)), 1000,
- new Node("b", new UUID(0xFEDCBA0987654321L, 0x1234567890ABCDEFL)), 2000
- );
-
- byte[] bytes = VersionedSerialization.toBytes(originalMap, serializer);
- Map<Node, Integer> restoredMap = VersionedSerialization.fromBytes(bytes, serializer);
-
- assertThat(restoredMap, equalTo(originalMap));
- }
-
- @Test
- void v1CanBeDeserialized() {
- byte[] bytes = Base64.getDecoder().decode("Ae++QwMB775DAmHvzauQeFY0EiFDZYcJutz+6QcB775DAmIhQ2WHCbrc/u/Nq5B4VjQS0Q8=");
- Map<Node, Integer> restoredMap = VersionedSerialization.fromBytes(bytes, serializer);
-
- assertThat(restoredMap, is(aMapWithSize(2)));
- assertThat(restoredMap, hasEntry(new Node("a", new UUID(0x1234567890ABCDEFL, 0xFEDCBA0987654321L)), 1000));
- assertThat(restoredMap, hasEntry(new Node("b", new UUID(0xFEDCBA0987654321L, 0x1234567890ABCDEFL)), 2000));
- }
-}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerAlterFilterTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerAlterFilterTest.java
index d0173122e44..ffbb358c029 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerAlterFilterTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerAlterFilterTest.java
@@ -22,21 +22,14 @@ import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesFromManager;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doAnswer;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalNode;
-import org.apache.ignite.internal.metastorage.server.If;
import org.apache.ignite.internal.network.ClusterNodeImpl;
import org.apache.ignite.network.NetworkAddress;
import org.junit.jupiter.params.ParameterizedTest;
@@ -156,16 +149,13 @@ public class DistributionZoneManagerAlterFilterTest extends BaseDistributionZone
int zoneId = getZoneId(zoneName);
- if (ZONE_NAME.equals(zoneName)) {
- assertNull(distributionZoneManager.zonesState().get(zoneId).scaleDownTask());
- }
-
topology.removeNodes(Set.of(C));
// Check that scale down task was scheduled.
assertTrue(
waitForCondition(
- () -> distributionZoneManager.zonesState().get(zoneId).scaleDownTask() != null,
+ () -> distributionZoneManager.dataNodesManager().zoneTimers(zoneId) != null
+ && distributionZoneManager.dataNodesManager().zoneTimers(zoneId).scaleDown.taskIsScheduled(),
ZONE_MODIFICATION_AWAIT_TIMEOUT
)
);
@@ -175,18 +165,12 @@ public class DistributionZoneManagerAlterFilterTest extends BaseDistributionZone
alterZone(zoneName, null, null, newFilter);
- // Node C is still in data nodes because altering a filter triggers only immediate scale up.
- assertDataNodesFromManager(distributionZoneManager, metaStorageManager::appliedRevision, catalogManager::latestCatalogVersion,
- zoneId, Set.of(C, D), ZONE_MODIFICATION_AWAIT_TIMEOUT);
-
- // Check that scale down task is still scheduled.
- assertNotNull(distributionZoneManager.zonesState().get(zoneId).scaleUpTask());
-
- // Alter zone so we could check that node C is removed from data nodes.
- alterZone(zoneName, INFINITE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, newFilter);
-
+ // Node C is removed from data nodes instantly.
assertDataNodesFromManager(distributionZoneManager, metaStorageManager::appliedRevision, catalogManager::latestCatalogVersion,
zoneId, Set.of(D), ZONE_MODIFICATION_AWAIT_TIMEOUT);
+
+ // Check that scale down task is not scheduled.
+ assertTrue(distributionZoneManager.dataNodesManager().zoneTimers(zoneId).scaleDown.taskIsCancelled());
}
/**
@@ -206,16 +190,13 @@ public class DistributionZoneManagerAlterFilterTest extends BaseDistributionZone
int zoneId = getZoneId(zoneName);
- if (ZONE_NAME.equals(zoneName)) {
- assertNull(distributionZoneManager.zonesState().get(zoneId).scaleUpTask());
- }
-
topology.putNode(D);
// Check that scale up task was scheduled.
assertTrue(
waitForCondition(
- () -> distributionZoneManager.zonesState().get(zoneId).scaleUpTask() != null,
+ () -> distributionZoneManager.dataNodesManager().zoneTimers(zoneId) != null
+ && distributionZoneManager.dataNodesManager().zoneTimers(zoneId).scaleUp.taskIsScheduled(),
ZONE_MODIFICATION_AWAIT_TIMEOUT
)
);
@@ -232,26 +213,16 @@ public class DistributionZoneManagerAlterFilterTest extends BaseDistributionZone
List.of(DEFAULT_STORAGE_PROFILE)
);
- doAnswer(invocation -> {
- If iif = invocation.getArgument(0);
-
- // Emulate a situation when immediate timer was run after filter altering and new node was added, so timer was scheduled.
- byte[] key = zoneScaleUpChangeTriggerKey(zoneId).bytes();
-
- if (Arrays.stream(iif.cond().keys()).anyMatch(k -> Arrays.equals(key, k))) {
- assertNotNull(distributionZoneManager.zonesState().get(zoneId).scaleUpTask());
+ topology.putNode(e);
- topology.putNode(e);
- }
- return invocation.callRealMethod();
- }).when(keyValueStorage).invoke(any(), any(), any());
-
- // Check that node E, that was added while filter's altering, is not propagated to data nodes.
+ // Check that node E, that was added while filter's altering, is not instantly propagated to data nodes.
assertDataNodesFromManager(distributionZoneManager, metaStorageManager::appliedRevision, catalogManager::latestCatalogVersion,
zoneId, Set.of(C, D), ZONE_MODIFICATION_AWAIT_TIMEOUT);
- // Assert that scheduled timer was not canceled because of immediate scale up after filter altering.
- assertNotNull(distributionZoneManager.zonesState().get(zoneId).scaleUpTask());
+ // Assert that scheduled timer was created because of node E addition.
+ assertTrue(
+ waitForCondition(() -> distributionZoneManager.dataNodesManager().zoneTimers(zoneId).scaleUp.taskIsScheduled(), 2000)
+ );
alterZone(zoneName, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, newFilter);
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
index 30889c570b4..94f2cb2c663 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
@@ -22,8 +22,9 @@ import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesFromLogicalNodesInStorage;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertLogicalTopology;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.logicalNodeFromNode;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownTimerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpTimerKey;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -31,7 +32,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Set;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalNode;
-import org.apache.ignite.network.NetworkAddress;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
@@ -43,7 +43,7 @@ public class DistributionZoneManagerConfigurationChangesTest extends BaseDistrib
private static final String NEW_ZONE_NAME = "zone2";
- private static final LogicalNode NODE_1 = new LogicalNode(randomUUID(), "node1", new NetworkAddress("localhost", 123));
+ private static final LogicalNode NODE_1 = logicalNodeFromNode(new Node("node1", randomUUID()));
private static final Set<LogicalNode> nodes = Set.of(NODE_1);
@@ -82,7 +82,8 @@ public class DistributionZoneManagerConfigurationChangesTest extends BaseDistrib
dropZone(ZONE_NAME);
- assertZonesKeysInMetaStorage(zoneId, null);
+ // Data nodes should be removed from meta storage
+ assertZonesKeysInMetaStorage(zoneId, null, false);
}
@ParameterizedTest
@@ -100,19 +101,26 @@ public class DistributionZoneManagerConfigurationChangesTest extends BaseDistrib
}
private void assertZonesKeysInMetaStorage(int zoneId, @Nullable Set<LogicalNode> clusterNodes) throws InterruptedException {
- assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes, keyValueStorage);
+ assertZonesKeysInMetaStorage(zoneId, clusterNodes, true);
+ }
+
+ private void assertZonesKeysInMetaStorage(int zoneId, @Nullable Set<LogicalNode> clusterNodes, boolean checkNodes)
+ throws InterruptedException {
+ if (checkNodes) {
+ assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes, keyValueStorage);
+ }
if (clusterNodes != null) {
- assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value() != null, 5000));
- assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value() != null, 5000));
+ assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleUpTimerKey(zoneId).bytes()).value() != null, 5000));
+ assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleDownTimerKey(zoneId).bytes()).value() != null, 5000));
} else {
- assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value() == null, 5000));
- assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value() == null, 5000));
+ assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleUpTimerKey(zoneId).bytes()).value() == null, 5000));
+ assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleDownTimerKey(zoneId).bytes()).value() == null, 5000));
}
assertArrayEquals(
- keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value(),
- keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value()
+ keyValueStorage.get(zoneScaleUpTimerKey(zoneId).bytes()).value(),
+ keyValueStorage.get(zoneScaleDownTimerKey(zoneId).bytes()).value()
);
}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerScaleUpScaleDownTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerScaleUpScaleDownTest.java
index 3ffd46465c4..beb53fce7c7 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerScaleUpScaleDownTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerScaleUpScaleDownTest.java
@@ -17,50 +17,41 @@
package org.apache.ignite.internal.distributionzones;
+import static java.util.Collections.emptyMap;
import static java.util.UUID.randomUUID;
+import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesFromLogicalNodesInStorage;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesInStorage;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertLogicalTopology;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertZoneScaleDownChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertZoneScaleUpChangeTriggerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.dataNodeHistoryContext;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.logicalNodeFromNode;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.PARTITION_DISTRIBUTION_RESET_TIMEOUT;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.createZoneManagerExecutor;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
-import static org.apache.ignite.internal.metastorage.server.KeyValueUpdateContext.kvContext;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
-import static org.apache.ignite.internal.util.ByteUtils.bytesToLongKeepingOrder;
-import static org.apache.ignite.internal.util.ByteUtils.longToBytesKeepingOrder;
+import static org.apache.ignite.internal.util.CompletableFutures.falseCompletedFuture;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doAnswer;
-import java.util.Arrays;
-import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalNode;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.ZoneState;
+import org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DataNodesHistoryContext;
+import org.apache.ignite.internal.distributionzones.events.HaZoneTopologyUpdateEvent;
+import org.apache.ignite.internal.distributionzones.events.HaZoneTopologyUpdateEventParams;
import org.apache.ignite.internal.hlc.HybridTimestamp;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
-import org.apache.ignite.internal.metastorage.server.If;
-import org.apache.ignite.internal.thread.NamedThreadFactory;
-import org.apache.ignite.internal.thread.StripedScheduledThreadPoolExecutor;
-import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
import org.apache.ignite.network.NetworkAddress;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.Test;
@@ -71,12 +62,6 @@ import org.junit.jupiter.api.Test;
public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributionZoneManagerTest {
private static final IgniteLogger LOG = Loggers.forClass(DistributionZoneManagerScaleUpScaleDownTest.class);
- private static final LogicalNode NODE_1 = new LogicalNode(randomUUID(), "node1", new NetworkAddress("localhost", 123));
-
- private static final LogicalNode NODE_2 = new LogicalNode(randomUUID(), "node2", new NetworkAddress("localhost", 123));
-
- private static final LogicalNode NODE_3 = new LogicalNode(randomUUID(), "node3", new NetworkAddress("localhost", 123));
-
private static final Node A = new Node("A", randomUUID());
private static final Node B = new Node("B", randomUUID());
@@ -87,21 +72,27 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
private static final Node E = new Node("E", randomUUID());
+ private static final LogicalNode NODE_A = logicalNodeFromNode(A);
+
+ private static final LogicalNode NODE_B = logicalNodeFromNode(B);
+
+ private static final LogicalNode NODE_C = logicalNodeFromNode(C);
+
@Test
void testDataNodesPropagationAfterScaleUpTriggered() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- Set<LogicalNode> clusterNodes = Set.of(NODE_1);
+ Set<LogicalNode> clusterNodes = Set.of(NODE_A);
int defaultZoneId = getDefaultZone().id();
assertDataNodesFromLogicalNodesInStorage(defaultZoneId, clusterNodes, keyValueStorage);
- topology.putNode(NODE_2);
+ topology.putNode(NODE_B);
- Set<LogicalNode> clusterNodes2 = Set.of(NODE_1, NODE_2);
+ Set<LogicalNode> clusterNodes2 = Set.of(NODE_A, NODE_B);
assertLogicalTopology(clusterNodes2, keyValueStorage);
@@ -111,9 +102,9 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes2, keyValueStorage);
- topology.putNode(NODE_3);
+ topology.putNode(NODE_C);
- Set<LogicalNode> clusterNodes3 = Set.of(NODE_1, NODE_2, NODE_3);
+ Set<LogicalNode> clusterNodes3 = Set.of(NODE_A, NODE_B, NODE_C);
assertLogicalTopology(clusterNodes3, keyValueStorage);
@@ -129,21 +120,21 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
createZone(ZONE_NAME, IMMEDIATE_TIMER_VALUE, null, null);
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- assertDataNodesFromLogicalNodesInStorage(getDefaultZone().id(), Set.of(NODE_1), keyValueStorage);
- assertDataNodesFromLogicalNodesInStorage(getZoneId(ZONE_NAME), Set.of(NODE_1), keyValueStorage);
+ assertDataNodesFromLogicalNodesInStorage(getDefaultZone().id(), Set.of(NODE_A), keyValueStorage);
+ assertDataNodesFromLogicalNodesInStorage(getZoneId(ZONE_NAME), Set.of(NODE_A), keyValueStorage);
}
@Test
void testDataNodesPropagationAfterScaleDownTriggered() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- topology.putNode(NODE_2);
+ topology.putNode(NODE_B);
- Set<LogicalNode> clusterNodes = Set.of(NODE_1, NODE_2);
+ Set<LogicalNode> clusterNodes = Set.of(NODE_A, NODE_B);
int defaultZoneId = getDefaultZone().id();
@@ -155,9 +146,9 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes, keyValueStorage);
- topology.removeNodes(Set.of(NODE_2));
+ topology.removeNodes(Set.of(NODE_B));
- Set<LogicalNode> clusterNodes2 = Set.of(NODE_1);
+ Set<LogicalNode> clusterNodes2 = Set.of(NODE_A);
assertLogicalTopology(clusterNodes2, keyValueStorage);
@@ -171,9 +162,9 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
void testDataNodesPropagationForDefaultZoneAfterScaleUpTriggered() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- Set<LogicalNode> clusterNodes = Set.of(NODE_1);
+ Set<LogicalNode> clusterNodes = Set.of(NODE_A);
CatalogZoneDescriptor defaultZone = getDefaultZone();
@@ -181,9 +172,9 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
alterZone(defaultZone.name(), INFINITE_TIMER_VALUE, INFINITE_TIMER_VALUE, null);
- topology.putNode(NODE_2);
+ topology.putNode(NODE_B);
- Set<LogicalNode> clusterNodes2 = Set.of(NODE_1, NODE_2);
+ Set<LogicalNode> clusterNodes2 = Set.of(NODE_A, NODE_B);
assertLogicalTopology(clusterNodes2, keyValueStorage);
@@ -196,11 +187,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
void testDataNodesPropagationForDefaultZoneAfterScaleDownTriggered() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- topology.putNode(NODE_2);
+ topology.putNode(NODE_B);
- Set<LogicalNode> clusterNodes = Set.of(NODE_1, NODE_2);
+ Set<LogicalNode> clusterNodes = Set.of(NODE_A, NODE_B);
CatalogZoneDescriptor defaultZone = getDefaultZone();
@@ -208,9 +199,9 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
alterZone(defaultZone.name(), INFINITE_TIMER_VALUE, INFINITE_TIMER_VALUE, null);
- topology.removeNodes(Set.of(NODE_2));
+ topology.removeNodes(Set.of(NODE_B));
- Set<LogicalNode> clusterNodes2 = Set.of(NODE_1);
+ Set<LogicalNode> clusterNodes2 = Set.of(NODE_A);
assertLogicalTopology(clusterNodes2, keyValueStorage);
@@ -223,11 +214,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
void testDropZoneDoNotPropagateDataNodesAfterScaleUp() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- topology.putNode(NODE_2);
+ topology.putNode(NODE_B);
- Set<LogicalNode> clusterNodes2 = Set.of(NODE_1, NODE_2);
+ Set<LogicalNode> clusterNodes2 = Set.of(NODE_A, NODE_B);
assertLogicalTopology(clusterNodes2, keyValueStorage);
@@ -237,29 +228,30 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes2, keyValueStorage);
- assertNotNull(keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value(),
- "zoneScaleUpChangeTriggerKey must be not null.");
- assertNotNull(keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value(),
- "zoneScaleDownChangeTriggerKey must be not null.");
+ DataNodesHistoryContext context = dataNodeHistoryContext(metaStorageManager, zoneId);
+ assertTrue(context.scaleUpTimerPresent());
+ assertTrue(context.scaleDownTimerPresent());
dropZone(ZONE_NAME);
- assertDataNodesFromLogicalNodesInStorage(zoneId, null, keyValueStorage);
- assertZoneScaleUpChangeTriggerKey(null, zoneId, keyValueStorage);
- assertZoneScaleDownChangeTriggerKey(null, zoneId, keyValueStorage);
+ // Data nodes history should not be dropped after zone drop. Deferred removal should happen on LWM move.
+ assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes2, keyValueStorage);
+ context = dataNodeHistoryContext(metaStorageManager, zoneId);
+ assertFalse(context.scaleUpTimerPresent());
+ assertFalse(context.scaleDownTimerPresent());
}
@Test
void testDropZoneDoNotPropagateDataNodesAfterScaleDown() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- topology.putNode(NODE_2);
+ topology.putNode(NODE_B);
- topology.removeNodes(Set.of(NODE_2));
+ topology.removeNodes(Set.of(NODE_B));
- Set<LogicalNode> clusterNodes2 = Set.of(NODE_1);
+ Set<LogicalNode> clusterNodes2 = Set.of(NODE_A);
assertLogicalTopology(clusterNodes2, keyValueStorage);
@@ -269,16 +261,17 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes2, keyValueStorage);
- assertNotNull(keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value(),
- "zoneScaleUpChangeTriggerKey must be not null.");
- assertNotNull(keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value(),
- "zoneScaleDownChangeTriggerKey must be not null.");
+ DataNodesHistoryContext context = dataNodeHistoryContext(metaStorageManager, zoneId);
+ assertTrue(context.scaleUpTimerPresent());
+ assertTrue(context.scaleDownTimerPresent());
dropZone(ZONE_NAME);
- assertDataNodesFromLogicalNodesInStorage(zoneId, null, keyValueStorage);
- assertZoneScaleUpChangeTriggerKey(null, zoneId, keyValueStorage);
- assertZoneScaleDownChangeTriggerKey(null, zoneId, keyValueStorage);
+ // Data nodes history should not be dropped after zone drop. Deferred removal should happen on LWM move.
+ assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes2, keyValueStorage);
+ context = dataNodeHistoryContext(metaStorageManager, zoneId);
+ assertFalse(context.scaleUpTimerPresent());
+ assertFalse(context.scaleDownTimerPresent());
}
@Test
@@ -291,11 +284,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(), keyValueStorage);
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
+ assertLogicalTopology(Set.of(NODE_A), keyValueStorage);
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
+ assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_A), keyValueStorage);
}
@Test
@@ -308,34 +301,34 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(), keyValueStorage);
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
+ assertLogicalTopology(Set.of(NODE_A), keyValueStorage);
assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(), keyValueStorage);
alterZone(ZONE_NAME, IMMEDIATE_TIMER_VALUE, null, null);
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
+ assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_A), keyValueStorage);
}
@Test
void testUpdateZoneScaleDownTriggersDataNodePropagation() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
+ topology.putNode(NODE_A);
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
+ assertLogicalTopology(Set.of(NODE_A), keyValueStorage);
createZone(ZONE_NAME, null, 100, null);
int zoneId = getZoneId(ZONE_NAME);
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
+ assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_A), keyValueStorage);
- topology.removeNodes(Set.of(NODE_1));
+ topology.removeNodes(Set.of(NODE_A));
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
+ assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_A), keyValueStorage);
alterZone(ZONE_NAME, null, IMMEDIATE_TIMER_VALUE, null);
@@ -343,146 +336,36 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
}
@Test
- void testCleanUpAfterSchedulers() throws Exception {
+ void testHistory() throws Exception {
preparePrerequisites();
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(E), 1004);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1003);
-
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1003L));
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1004L));
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1007L));
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1004);
+ HybridTimestamp time = clock.now();
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1003L));
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1004L));
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1007L));
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- assertFalse(zoneState.topologyAugmentationMap().containsKey(1003L));
- assertFalse(zoneState.topologyAugmentationMap().containsKey(1004L));
- assertTrue(zoneState.topologyAugmentationMap().containsKey(1007L));
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1015);
-
- assertTrue(zoneState.topologyAugmentationMap().isEmpty());
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, D, E), keyValueStorage);
- }
-
- @Test
- void testScaleUpSetToMaxInt() throws Exception {
- startDistributionZoneManager();
-
- createZone(ZONE_NAME, 100, null, null);
+ LOG.info("Time: {}", time);
int zoneId = getZoneId(ZONE_NAME);
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(), keyValueStorage);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- assertNull(zoneState.scaleUpTask());
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
- topology.putNode(NODE_1);
+ addNodes(Set.of(D));
+ addNodes(Set.of(E));
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, D, E), keyValueStorage);
- assertTrue(waitForCondition(() -> zoneState.scaleUpTask() != null, 1000L));
+ removeNodes(Set.of(C));
- alterZone(ZONE_NAME, INFINITE_TIMER_VALUE, null, null);
-
- assertTrue(waitForCondition(() -> zoneState.scaleUpTask().isCancelled(), 1000L));
- }
-
- @Test
- void testScaleDownSetToMaxInt() throws Exception {
- startDistributionZoneManager();
-
- topology.putNode(NODE_1);
-
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
-
- createZone(ZONE_NAME, null, 100, null);
-
- int zoneId = getZoneId(ZONE_NAME);
-
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- assertNull(zoneState.scaleDownTask());
-
- topology.removeNodes(Set.of(NODE_1));
-
- assertTrue(waitForCondition(() -> zoneState.scaleDownTask() != null, 1000L));
-
- alterZone(ZONE_NAME, null, INFINITE_TIMER_VALUE, null);
-
- assertTrue(waitForCondition(() -> zoneState.scaleDownTask().isCancelled(), 1000L));
- }
-
- @Test
- void testScaleUpDidNotChangeDataNodesWhenTriggerKeyWasConcurrentlyChanged() throws Exception {
- startDistributionZoneManager();
-
- createZone(ZONE_NAME, IMMEDIATE_TIMER_VALUE, null, null);
-
- int zoneId = getZoneId(ZONE_NAME);
-
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(), keyValueStorage);
-
- assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value() != null, 5000));
-
- long revisionOfScaleUp = bytesToLongKeepingOrder(keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value());
-
- doAnswer(invocation -> {
- If iif = invocation.getArgument(0);
-
- // Emulate a situation when one of the scale up keys gets concurrently updated during a Meta Storage invoke. We then expect
- // that the invoke call will be retried.
- byte[] keyScaleUp = zoneScaleUpChangeTriggerKey(zoneId).bytes();
- byte[] keyDataNodes = zoneDataNodesKey(zoneId).bytes();
-
- if (Arrays.stream(iif.cond().keys()).anyMatch(k -> Arrays.equals(keyScaleUp, k))) {
- byte[] value = keyValueStorage.get(zoneDataNodesKey(zoneId).bytes()).value();
-
- keyValueStorage.putAll(
- List.of(keyScaleUp, keyDataNodes),
- List.of(longToBytesKeepingOrder(revisionOfScaleUp + 100L), value),
- kvContext(HybridTimestamp.MIN_VALUE)
- );
- }
-
- return invocation.callRealMethod();
- }).when(keyValueStorage).invoke(any(), any(), any());
-
- topology.putNode(NODE_1);
-
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
-
- assertZoneScaleUpChangeTriggerKey(revisionOfScaleUp + 100L, zoneId, keyValueStorage);
-
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, D, E), keyValueStorage);
}
@Test
void testPartitionDistributionResetTaskScheduling() throws Exception {
startDistributionZoneManager();
- topology.putNode(NODE_1);
- topology.putNode(NODE_2);
- topology.putNode(NODE_3);
+ topology.putNode(NODE_A);
+ topology.putNode(NODE_B);
+ topology.putNode(NODE_C);
- assertLogicalTopology(Set.of(NODE_1, NODE_2, NODE_3), keyValueStorage);
+ assertLogicalTopology(Set.of(NODE_A, NODE_B, NODE_C), keyValueStorage);
String haZoneName = "haZone";
String scZoneName = "scZone";
@@ -493,22 +376,28 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int haZoneId = getZoneId(haZoneName);
int scZoneId = getZoneId(scZoneName);
- topology.removeNodes(Set.of(NODE_3));
+ List<HaZoneTopologyUpdateEventParams> eventList = new CopyOnWriteArrayList<>();
+
+ distributionZoneManager.listen(HaZoneTopologyUpdateEvent.TOPOLOGY_REDUCED, event -> {
+ eventList.add(event);
+
+ return falseCompletedFuture();
+ });
- assertLogicalTopology(Set.of(NODE_1, NODE_2), keyValueStorage);
+ topology.removeNodes(Set.of(NODE_C));
+
+ assertLogicalTopology(Set.of(NODE_A, NODE_B), keyValueStorage);
long topologyRevision = metaStorageManager.get(
zonesLogicalTopologyKey()).get(1, TimeUnit.SECONDS).revision();
assertTrue(waitForCondition(() -> metaStorageManager.appliedRevision() >= topologyRevision, 1000));
- assertNull(distributionZoneManager.zonesState().get(scZoneId).partitionDistributionResetTask());
+ assertFalse(distributionZoneManager.dataNodesManager().zoneTimers(scZoneId).partitionReset.taskIsScheduled());
- assertNotNull(distributionZoneManager.zonesState().get(haZoneId).partitionDistributionResetTask());
- assertEquals(
- 0,
- distributionZoneManager.zonesState().get(haZoneId).partitionDistributionResetTask().getDelay(TimeUnit.SECONDS)
- );
+ assertTrue(waitForCondition(() -> !eventList.isEmpty(), 2000));
+ assertEquals(1, eventList.size());
+ assertEquals(haZoneId, eventList.get(0).zoneId());
// We don't want trigger stop of the timer, so, the target configuration must be smaller than Integer.MAX_VALUE
CompletableFuture<Void> changeConfigFuture = systemDistributedConfiguration.change(c0 -> c0.changeProperties()
@@ -516,65 +405,12 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
);
assertThat(changeConfigFuture, willCompleteSuccessfully());
- topology.removeNodes(Set.of(NODE_2));
+ topology.removeNodes(Set.of(NODE_B));
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
+ assertLogicalTopology(Set.of(NODE_A), keyValueStorage);
- assertTrue(waitForCondition(
- () -> distributionZoneManager.zonesState().get(haZoneId).partitionDistributionResetTask().getDelay(TimeUnit.SECONDS) > 0,
- 1000));
- // Not HA zone is not changed
- assertNull(distributionZoneManager.zonesState().get(scZoneId).partitionDistributionResetTask());
- }
-
- @Test
- void testScaleDownDidNotChangeDataNodesWhenTriggerKeyWasConcurrentlyChanged() throws Exception {
- startDistributionZoneManager();
-
- topology.putNode(NODE_1);
-
- assertLogicalTopology(Set.of(NODE_1), keyValueStorage);
-
- int defaultZoneId = getDefaultZone().id();
-
- assertDataNodesFromLogicalNodesInStorage(defaultZoneId, Set.of(NODE_1), keyValueStorage);
-
- createZone(ZONE_NAME, null, IMMEDIATE_TIMER_VALUE, null);
-
- int zoneId = getZoneId(ZONE_NAME);
-
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
-
- assertTrue(waitForCondition(() -> keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value() != null, 5000));
-
- long revisionOfScaleDown = bytesToLongKeepingOrder(keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value());
-
- doAnswer(invocation -> {
- If iif = invocation.getArgument(0);
-
- // Emulate a situation when one of the scale down keys gets concurrently updated during a Meta Storage invoke. We then expect
- // that the invoke call will be retried.
- byte[] keyScaleDown = zoneScaleDownChangeTriggerKey(zoneId).bytes();
- byte[] keyDataNodes = zoneDataNodesKey(zoneId).bytes();
-
- if (Arrays.stream(iif.cond().keys()).anyMatch(k -> Arrays.equals(keyScaleDown, k))) {
- byte[] scaleDownRevision = longToBytesKeepingOrder(revisionOfScaleDown + 100);
-
- keyValueStorage.putAll(
- List.of(keyScaleDown, keyDataNodes),
- List.of(scaleDownRevision, keyValueStorage.get(zoneDataNodesKey(zoneId).bytes()).value()),
- kvContext(HybridTimestamp.MIN_VALUE)
- );
- }
-
- return invocation.callRealMethod();
- }).when(keyValueStorage).invoke(any(), any(), any());
-
- topology.removeNodes(Set.of(NODE_1));
-
- assertDataNodesFromLogicalNodesInStorage(zoneId, Set.of(NODE_1), keyValueStorage);
-
- assertZoneScaleDownChangeTriggerKey(revisionOfScaleDown + 100L, zoneId, keyValueStorage);
+ assertEquals(1, eventList.size());
+ assertFalse(distributionZoneManager.dataNodesManager().zoneTimers(scZoneId).partitionReset.taskIsScheduled());
}
@Test
@@ -583,15 +419,10 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1007);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(C));
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1003);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, D), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, D), keyValueStorage);
}
@Test
@@ -600,49 +431,10 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1007);
+ removeNodes(Set.of(C));
+ addNodes(Set.of(D));
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1003);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, D), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios1_3() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1003);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, D), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios1_4() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1003);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, D), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, D), keyValueStorage);
}
@Test
@@ -651,16 +443,10 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1003);
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
}
@Test
@@ -669,49 +455,10 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
+ removeNodes(Set.of(C));
+ addNodes(Set.of(C));
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1003);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios2_3() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1003);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios2_4() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1003);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
}
@Test
@@ -720,16 +467,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
+ addNodes(Set.of(D));
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
}
@Test
@@ -738,16 +480,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
+ addNodes(Set.of(D));
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
}
@Test
@@ -756,18 +493,14 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
+ addNodes(Set.of(D));
- zoneState.nodesToAddToDataNodes(Set.of(D), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
}
@Test
@@ -776,52 +509,14 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
+ removeNodes(Set.of(C));
+ addNodes(Set.of(C));
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1009);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C), keyValueStorage);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
+ removeNodes(Set.of(C));
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios3_5() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios3_6() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
}
@Test
@@ -830,17 +525,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
+ addNodes(Set.of(E));
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
- zoneState.nodesToAddToDataNodes(Set.of(E), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, E), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, E), keyValueStorage);
}
@Test
@@ -849,16 +538,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
- zoneState.nodesToAddToDataNodes(Set.of(E), 1009);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
+ addNodes(Set.of(E));
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, E), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, E), keyValueStorage);
}
@Test
@@ -867,70 +551,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- zoneState.nodesToAddToDataNodes(Set.of(E), 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, E), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios4_4() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
- zoneState.nodesToRemoveFromDataNodes(Set.of(B), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, C), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios4_5() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
- zoneState.nodesToRemoveFromDataNodes(Set.of(B), 1009);
+ removeNodes(Set.of(C));
+ addNodes(Set.of(C));
+ removeNodes(Set.of(B));
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, C), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios4_6() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(B), 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, C), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, C), keyValueStorage);
}
@Test
@@ -939,16 +564,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1007);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1009);
+ addNodes(Set.of(D));
+ removeNodes(Set.of(C));
+ addNodes(Set.of(C));
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
+ assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
}
@Test
@@ -957,120 +577,11 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1007);
- zoneState.nodesToAddToDataNodes(Set.of(C), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios5_3() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToAddToDataNodes(Set.of(D), 1003);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1007);
-
- zoneState.nodesToAddToDataNodes(Set.of(C), 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B, C, D), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios5_4() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1007);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios5_5() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1007);
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1009);
-
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
- }
-
- @Test
- void testVariousScaleUpScaleDownScenarios5_6() throws Exception {
- preparePrerequisites();
-
- int zoneId = getZoneId(ZONE_NAME);
-
- ZoneState zoneState = distributionZoneManager.zonesState().get(zoneId);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C), 1003);
- zoneState.nodesToAddToDataNodes(Set.of(D), 1007);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleUp(zoneId, 1007);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(D), 1009);
- distributionZoneManager.saveDataNodesToMetaStorageOnScaleDown(zoneId, 1009);
-
- DistributionZonesTestUtil.assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
- }
-
- @Test
- void testZoneStateAddRemoveNodesPreservesDuplicationsOfNodes() {
- StripedScheduledThreadPoolExecutor executor = createZoneManagerExecutor(
- 1,
- new NamedThreadFactory("test-dst-zones-scheduler", LOG)
- );
-
- try {
- ZoneState zoneState = new ZoneState(executor);
-
- zoneState.nodesToAddToDataNodes(Set.of(A, B), 1);
- zoneState.nodesToAddToDataNodes(Set.of(A, B), 2);
-
- List<Node> nodes = zoneState.nodesToBeAddedToDataNodes(0, 2);
+ removeNodes(Set.of(C));
+ addNodes(Set.of(D));
+ removeNodes(Set.of(D));
- nodes.sort(Comparator.comparing(Node::nodeName));
-
- assertEquals(List.of(A, A, B, B), nodes);
-
- zoneState.nodesToRemoveFromDataNodes(Set.of(C, D), 3);
- zoneState.nodesToRemoveFromDataNodes(Set.of(C, D), 4);
-
- nodes = zoneState.nodesToBeRemovedFromDataNodes(2, 4);
-
- nodes.sort(Comparator.comparing(Node::nodeName));
-
- assertEquals(List.of(C, C, D, D), nodes);
- } finally {
- IgniteUtils.shutdownAndAwaitTermination(executor, 10, TimeUnit.SECONDS);
- }
+ assertDataNodesInStorage(zoneId, Set.of(A, B), keyValueStorage);
}
/**
@@ -1085,15 +596,15 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
private void preparePrerequisites(@Nullable String filter) throws Exception {
startDistributionZoneManager();
- LogicalNode a = new LogicalNode(A.nodeId(), A.nodeName(), new NetworkAddress("localhost", 123));
+ LogicalNode a = logicalNodeFromNode(A);
- LogicalNode b = new LogicalNode(B.nodeId(), B.nodeName(), new NetworkAddress("localhost", 123));
+ LogicalNode b = logicalNodeFromNode(B);
- LogicalNode c = new LogicalNode(C.nodeId(), C.nodeName(), new NetworkAddress("localhost", 123));
+ LogicalNode c = logicalNodeFromNode(C);
- LogicalNode d = new LogicalNode(D.nodeId(), D.nodeName(), new NetworkAddress("localhost", 123));
+ LogicalNode d = logicalNodeFromNode(D);
- LogicalNode e = new LogicalNode(E.nodeId(), E.nodeName(), new NetworkAddress("localhost", 123));
+ LogicalNode e = logicalNodeFromNode(E);
topology.putNode(a);
topology.putNode(b);
@@ -1114,17 +625,6 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
int zoneId = getZoneId(ZONE_NAME);
assertDataNodesFromLogicalNodesInStorage(zoneId, clusterNodes, keyValueStorage);
-
- long scaleUpChangeTriggerKey = bytesToLongKeepingOrder(
- keyValueStorage.get(zoneScaleUpChangeTriggerKey(zoneId).bytes()).value()
- );
-
- long scaleDownChangeTriggerKey = bytesToLongKeepingOrder(
- keyValueStorage.get(zoneScaleDownChangeTriggerKey(zoneId).bytes()).value()
- );
-
- assertEquals(scaleUpChangeTriggerKey, scaleDownChangeTriggerKey,
- "zoneScaleUpChangeTriggerKey and zoneScaleDownChangeTriggerKey are not equal");
}
protected void createZone(
@@ -1137,4 +637,23 @@ public class DistributionZoneManagerScaleUpScaleDownTest extends BaseDistributio
consistencyMode
);
}
+
+ private void addNodes(Set<Node> nodes) {
+ for (Node n : nodes) {
+ topology.putNode(new LogicalNode(
+ new ClusterNodeImpl(n.nodeId(), n.nodeName(), new NetworkAddress("localhost", 123)),
+ emptyMap(),
+ emptyMap(),
+ List.of("default")
+ ));
+ }
+ }
+
+ private void removeNodes(Set<Node> nodes) {
+ topology.removeNodes(
+ nodes.stream()
+ .map(n -> new LogicalNode(n.nodeId(), n.nodeName(), new NetworkAddress("localhost", 123)))
+ .collect(toSet())
+ );
+ }
}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneTimerSerializerTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneTimerSerializerTest.java
new file mode 100644
index 00000000000..8c7f87f5cd6
--- /dev/null
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneTimerSerializerTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.distributionzones;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.apache.ignite.internal.distributionzones.DistributionZoneTimer.DistributionZoneTimerSerializer.deserialize;
+import static org.apache.ignite.internal.distributionzones.DistributionZoneTimer.DistributionZoneTimerSerializer.serialize;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.internal.distributionzones.DistributionZoneTimer.DistributionZoneTimerSerializer;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link DistributionZoneTimerSerializer}.
+ */
+public class DistributionZoneTimerSerializerTest {
+ private static final UUID NODE_A_ID = new UUID(0x1234567890ABCDEFL, 0xFEDCBA0987654321L);
+ private static final UUID NODE_B_ID = new UUID(0xFEDCBA0987654321L, 0x1234567890ABCDEFL);
+
+ private static final NodeWithAttributes A = new NodeWithAttributes("node_A", NODE_A_ID, emptyMap(), emptyList());
+ private static final NodeWithAttributes B = new NodeWithAttributes(
+ "node_B",
+ NODE_B_ID,
+ Map.of("region", "US"),
+ List.of("default")
+ );
+
+ private static final DistributionZoneTimer TIMER = new DistributionZoneTimer(new HybridTimestamp(12341234L, 10), 10000, Set.of(A, B));
+ private static final String SERIALIZED_V1 = "Ae++Q4uAyP/EF5FOAwHvvkMB775DB25vZGVfQe/Nq5B4VjQSIUNlhwm63P4BAQHvvkMB775DB25vZGVfQiFDZYc"
+ + "Jutz+782rkHhWNBICB3JlZ2lvbgNVUwIIZGVmYXVsdA==";
+
+ @Test
+ public void serializationAndDeserialization() {
+ DistributionZoneTimer timer0 = new DistributionZoneTimer(new HybridTimestamp(1234L, 1), 60, Set.of(A));
+ DistributionZoneTimer timer1 = new DistributionZoneTimer(new HybridTimestamp(12341234L, 10), 10000, Set.of(A, B));
+
+ DistributionZoneTimer deserializedTimer0 = deserialize(serialize(timer0));
+ assertEquals(timer0, deserializedTimer0);
+
+ DistributionZoneTimer deserializedTimer1 = deserialize(serialize(timer1));
+ assertEquals(timer1, deserializedTimer1);
+ }
+
+ @Test
+ public void v1CanBeDeserialized() {
+ byte[] serializedBytes = Base64.getDecoder().decode(SERIALIZED_V1);
+ DistributionZoneTimer deserializedTimer = deserialize(serializedBytes);
+ assertEquals(TIMER, deserializedTimer);
+ }
+}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZonesSchedulersTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZonesSchedulersTest.java
index 3d74cc6359c..4e2276c4929 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZonesSchedulersTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZonesSchedulersTest.java
@@ -28,9 +28,9 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
import java.util.function.Supplier;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.ZoneState;
-import org.apache.ignite.internal.lang.IgniteTriConsumer;
+import org.apache.ignite.internal.distributionzones.DataNodesManager.ZoneTimerSchedule;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
import org.apache.ignite.internal.thread.NamedThreadFactory;
@@ -59,32 +59,18 @@ public class DistributionZonesSchedulersTest {
}
@Test
- void testScaleUpSchedule() throws InterruptedException {
- ZoneState state = new ZoneState(executor);
+ void testSchedule() throws InterruptedException {
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
- testSchedule(state::rescheduleScaleUp);
+ testSchedule(zoneTimerSchedule::reschedule);
}
- @Test
- void testScaleDownSchedule() throws InterruptedException {
- ZoneState state = new ZoneState(executor);
-
- testSchedule(state::rescheduleScaleDown);
- }
-
- @Test
- void testPartitionDistributionResetSchedule() throws InterruptedException {
- ZoneState state = new ZoneState(executor);
-
- testSchedule(state::reschedulePartitionDistributionReset);
- }
-
- private static void testSchedule(IgniteTriConsumer<Long, Runnable, Integer> fn) throws InterruptedException {
+ private static void testSchedule(BiConsumer<Long, Runnable> scheduler) throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
assertEquals(1L, latch.getCount());
- fn.accept(0L, latch::countDown, STRIPE_0);
+ scheduler.accept(0L, latch::countDown);
latch.await();
@@ -92,38 +78,24 @@ public class DistributionZonesSchedulersTest {
}
@Test
- void testScaleUpReScheduling() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- testReScheduling(state::rescheduleScaleUp);
- }
-
- @Test
- void testScaleDownReScheduling() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
+ void testReScheduling() throws InterruptedException {
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
- testReScheduling(state::rescheduleScaleDown);
- }
-
- @Test
- void testPartitionDistributionResetScheduling() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- testReScheduling(state::reschedulePartitionDistributionReset);
+ testReScheduling(zoneTimerSchedule::reschedule);
}
/**
- * Tests that scaleUp/scaleDown/partitionDistributionReset tasks with a zero delay will not be canceled by other tasks.
- * Tests that scaleUp/scaleDown/partitionDistributionReset tasks with a delay grater then zero will be canceled by other tasks.
+ * Tests that tasks with a zero delay will not be canceled by other tasks.
+ * Tests that tasks with a delay grater then zero will be canceled by other tasks.
*/
- private static void testReScheduling(IgniteTriConsumer<Long, Runnable, Integer> fn) throws InterruptedException {
+ private static void testReScheduling(BiConsumer<Long, Runnable> rescheduler) throws InterruptedException {
AtomicInteger counter = new AtomicInteger();
CountDownLatch firstTaskLatch = new CountDownLatch(1);
CountDownLatch lastTaskLatch = new CountDownLatch(1);
- fn.accept(
+ rescheduler.accept(
0L,
() -> {
try {
@@ -133,30 +105,28 @@ public class DistributionZonesSchedulersTest {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- },
- STRIPE_0
+ }
);
- fn.accept(1L, counter::incrementAndGet, STRIPE_0);
+ rescheduler.accept(1L, counter::incrementAndGet);
- fn.accept(0L, counter::incrementAndGet, STRIPE_0);
+ rescheduler.accept(0L, counter::incrementAndGet);
- fn.accept(0L, counter::incrementAndGet, STRIPE_0);
+ rescheduler.accept(0L, counter::incrementAndGet);
- fn.accept(1L, counter::incrementAndGet, STRIPE_0);
+ rescheduler.accept(1L, counter::incrementAndGet);
- fn.accept(1L, counter::incrementAndGet, STRIPE_0);
+ rescheduler.accept(1L, counter::incrementAndGet);
- fn.accept(0L, counter::incrementAndGet, STRIPE_0);
+ rescheduler.accept(0L, counter::incrementAndGet);
- fn.accept(
+ rescheduler.accept(
0L,
() -> {
counter.incrementAndGet();
lastTaskLatch.countDown();
- },
- STRIPE_0
+ }
);
firstTaskLatch.countDown();
@@ -167,34 +137,20 @@ public class DistributionZonesSchedulersTest {
}
@Test
- void testScaleUpOrdering() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- testOrdering(state::rescheduleScaleUp);
- }
-
- @Test
- void testScaleDownOrdering() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- testOrdering(state::rescheduleScaleDown);
- }
-
- @Test
- void testPartitionDistributionResetOrdering() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
+ void testOrdering() throws InterruptedException {
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
- testOrdering(state::reschedulePartitionDistributionReset);
+ testOrdering(zoneTimerSchedule::reschedule);
}
- private static void testOrdering(IgniteTriConsumer<Long, Runnable, Integer> fn) throws InterruptedException {
+ private static void testOrdering(BiConsumer<Long, Runnable> scheduler) throws InterruptedException {
AtomicInteger counter = new AtomicInteger();
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean sequentialOrder = new AtomicBoolean(true);
- fn.accept(
+ scheduler.accept(
0L,
() -> {
try {
@@ -208,14 +164,13 @@ public class DistributionZonesSchedulersTest {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- },
- STRIPE_0
+ }
);
for (int i = 2; i < 11; i++) {
int j = i;
- fn.accept(
+ scheduler.accept(
0L,
() -> {
counter.incrementAndGet();
@@ -223,8 +178,7 @@ public class DistributionZonesSchedulersTest {
if (counter.get() != j) {
sequentialOrder.set(false);
}
- },
- STRIPE_0
+ }
);
}
@@ -237,106 +191,44 @@ public class DistributionZonesSchedulersTest {
}
@Test
- void testScaleUpReScheduleWhenTaskIsEnded() throws InterruptedException {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- testReScheduleWhenTaskIsEnded(state::rescheduleScaleUp);
- }
-
- @Test
- void testScaleDownReScheduleWhenTaskIsEnded() throws InterruptedException {
- ZoneState state = new ZoneState(executor);
-
- testReScheduleWhenTaskIsEnded(state::rescheduleScaleUp);
- }
-
- @Test
- void testPartitionDistributionResetReScheduleWhenTaskIsEnded() throws InterruptedException {
- ZoneState state = new ZoneState(executor);
+ void testReScheduleWhenTaskIsEnded() throws InterruptedException {
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
- testReScheduleWhenTaskIsEnded(state::reschedulePartitionDistributionReset);
+ testReScheduleWhenTaskIsEnded(zoneTimerSchedule::reschedule);
}
- private static void testReScheduleWhenTaskIsEnded(IgniteTriConsumer<Long, Runnable, Integer> fn) throws InterruptedException {
+ private static void testReScheduleWhenTaskIsEnded(BiConsumer<Long, Runnable> scheduler) throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean flag = new AtomicBoolean();
assertEquals(1L, latch.getCount());
- fn.accept(0L, latch::countDown, STRIPE_0);
+ scheduler.accept(0L, latch::countDown);
latch.await(1000, TimeUnit.MILLISECONDS);
- fn.accept(0L, () -> flag.set(true), STRIPE_0);
+ scheduler.accept(0L, () -> flag.set(true));
assertTrue(waitForCondition(() -> 0L == latch.getCount(), 1500));
assertTrue(waitForCondition(flag::get, 1500));
}
@Test
- void testCancelScaleUpTaskOnStopScaleUp() {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
+ void testCancelTaskOnStop() {
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
- testCancelTask(state::rescheduleScaleUp, state::stopScaleUp, () -> state.scaleUpTask().isCancelled());
+ testCancelTask(zoneTimerSchedule::reschedule, zoneTimerSchedule::stopScheduledTask, zoneTimerSchedule::taskIsCancelled);
}
- @Test
- void testCancelScaleDownTaskOnStopScaleDown() {
- ZoneState state = new ZoneState(executor);
-
- testCancelTask(state::rescheduleScaleDown, state::stopScaleDown, () -> state.scaleDownTask().isCancelled());
- }
-
- @Test
- void testCancelPartitionDistributionResetTaskOnStopScaleDown() {
- ZoneState state = new ZoneState(executor);
-
- testCancelTask(
- state::reschedulePartitionDistributionReset,
- state::stopPartitionDistributionReset,
- () -> state.partitionDistributionResetTask().isCancelled()
- );
- }
-
- @Test
- void testCancelScaleUpTasksOnStopTimers() {
- ZoneState state = new ZoneState(executor);
-
- testCancelTask(state::rescheduleScaleUp, state::stopTimers, () -> state.scaleUpTask().isCancelled());
- }
-
- @Test
- void testCancelScaleDownTasksOnStopTimers() {
- ZoneState state = new ZoneState(executor);
-
- testCancelTask(state::rescheduleScaleDown, state::stopTimers, () -> state.scaleDownTask().isCancelled());
- }
-
- @Test
- void testCancelPartitionDistributionResetTasksOnStopTimers() {
- ZoneState state = new ZoneState(executor);
-
- testCancelTask(
- state::reschedulePartitionDistributionReset,
- state::stopTimers,
- () -> state.partitionDistributionResetTask().isCancelled()
- );
- }
-
- /**
- * {@link ZoneState#stopScaleUp()}, {@link ZoneState#stopScaleDown()}, {@link ZoneState#stopPartitionDistributionReset()}
- * and {@link ZoneState#stopTimers()} cancel task
- * if it is not started and has a delay greater than zero.
- */
private static void testCancelTask(
- IgniteTriConsumer<Long, Runnable, Integer> fn,
+ BiConsumer<Long, Runnable> scheduler,
Runnable stopTask,
Supplier<Boolean> isTaskCancelled
) {
CountDownLatch latch = new CountDownLatch(1);
- fn.accept(
+ scheduler.accept(
0L,
() -> {
try {
@@ -344,11 +236,10 @@ public class DistributionZonesSchedulersTest {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- },
- STRIPE_0
+ }
);
- fn.accept(1L, () -> {}, STRIPE_0);
+ scheduler.accept(1L, () -> {});
assertFalse(isTaskCancelled.get());
@@ -360,45 +251,27 @@ public class DistributionZonesSchedulersTest {
}
@Test
- void testNotCancelScaleUpTaskOnStopScaleUp() {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- testNotCancelTask(state::rescheduleScaleUp, state::stopScaleUp, () -> state.scaleUpTask().isCancelled());
- }
-
- @Test
- void testNotCancelScaleDownTaskOnStopScaleDown() {
- ZoneState state = new ZoneState(executor);
-
- testNotCancelTask(state::rescheduleScaleDown, state::stopScaleDown, () -> state.scaleDownTask().isCancelled());
-
- }
-
- @Test
- void testNotCancelPartitionDistributionResetTaskOnStopPartitionDistributionReset() {
- ZoneState state = new ZoneState(executor);
-
- testNotCancelTask(
- state::reschedulePartitionDistributionReset,
- state::stopPartitionDistributionReset,
- () -> state.partitionDistributionResetTask().isCancelled()
- );
+ void testNotCancelTaskOnStop() {
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
+ testRunAndCancelTask(zoneTimerSchedule::reschedule, zoneTimerSchedule::stopScheduledTask, zoneTimerSchedule::taskIsCancelled);
}
/**
- * {@link ZoneState#stopScaleUp()}, {@link ZoneState#stopScaleDown()} and {@link ZoneState#stopPartitionDistributionReset()}
- * doesn't cancel task
- * if it is not started and has a delay equal to zero.
+ * {@link ZoneTimerSchedule#stopScheduledTask()} doesn't cancel task if it is not started and has a delay equal to zero.
+ *
+ * @param rescheduler Rescheduler function, accepting delay and task.
+ * @param stopTask Stop task function.
+ * @param isTaskCancelled Task cancellation check function.
*/
- private static void testNotCancelTask(
- IgniteTriConsumer<Long, Runnable, Integer> fn,
+ private static void testRunAndCancelTask(
+ BiConsumer<Long, Runnable> rescheduler,
Runnable stopTask,
Supplier<Boolean> isTaskCancelled
) {
CountDownLatch latch = new CountDownLatch(1);
- fn.accept(
+ rescheduler.accept(
0L,
() -> {
try {
@@ -406,11 +279,10 @@ public class DistributionZonesSchedulersTest {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- },
- STRIPE_0
+ }
);
- fn.accept(0L, () -> {}, STRIPE_0);
+ rescheduler.accept(0L, () -> {});
assertFalse(isTaskCancelled.get());
@@ -422,75 +294,34 @@ public class DistributionZonesSchedulersTest {
}
/**
- * {@link ZoneState#stopTimers()} cancel task if it is not started and has a delay equal to zero.
+ * {@link ZoneTimerSchedule#stopScheduledTask()} ()} cancel task if it is not started and has a delay equal to zero.
*/
@Test
public void testCancelTasksOnStopTimersAndImmediateTimerValues() {
- ZoneState state = new DistributionZoneManager.ZoneState(executor);
-
- CountDownLatch scaleUpTaskLatch = new CountDownLatch(1);
-
- state.rescheduleScaleUp(
- 0L,
- () -> {
- try {
- assertTrue(scaleUpTaskLatch.await(3, TimeUnit.SECONDS));
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- },
- STRIPE_0
- );
-
- state.rescheduleScaleUp(0L, () -> {}, STRIPE_0);
-
- assertFalse(state.scaleUpTask().isCancelled());
+ ZoneTimerSchedule zoneTimerSchedule = new ZoneTimerSchedule(STRIPE_0, executor);
- CountDownLatch scaleDownTaskLatch = new CountDownLatch(1);
-
- state.rescheduleScaleDown(
- 0L,
- () -> {
- try {
- assertTrue(scaleDownTaskLatch.await(3, TimeUnit.SECONDS));
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- },
- STRIPE_0
- );
-
- state.rescheduleScaleDown(0L, () -> {}, STRIPE_0);
-
- assertFalse(state.scaleDownTask().isCancelled());
-
- CountDownLatch partitionDistributionResetTaskLatch = new CountDownLatch(1);
+ CountDownLatch latch = new CountDownLatch(1);
- state.reschedulePartitionDistributionReset(
+ zoneTimerSchedule.reschedule(
0L,
() -> {
try {
- assertTrue(partitionDistributionResetTaskLatch.await(3, TimeUnit.SECONDS));
+ assertTrue(latch.await(3, TimeUnit.SECONDS));
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- },
- STRIPE_0
+ }
);
- state.reschedulePartitionDistributionReset(0L, () -> {}, STRIPE_0);
+ zoneTimerSchedule.reschedule(0L, () -> {});
- assertFalse(state.partitionDistributionResetTask().isCancelled());
+ assertFalse(zoneTimerSchedule.taskIsCancelled());
- state.stopTimers();
+ zoneTimerSchedule.stopTimer();
- assertTrue(state.scaleUpTask().isCancelled());
- assertTrue(state.scaleDownTask().isCancelled());
- assertTrue(state.partitionDistributionResetTask().isCancelled());
+ assertTrue(zoneTimerSchedule.taskIsCancelled());
- scaleUpTaskLatch.countDown();
- scaleDownTaskLatch.countDown();
- partitionDistributionResetTaskLatch.countDown();
+ latch.countDown();
}
/**
@@ -498,13 +329,13 @@ public class DistributionZonesSchedulersTest {
*/
@Test
public void testTasksForTwoZonesDoesNotBlockEachOther() throws Exception {
- ZoneState state0 = new DistributionZoneManager.ZoneState(executor);
- ZoneState state1 = new DistributionZoneManager.ZoneState(executor);
+ ZoneTimerSchedule zoneTimerSchedule0 = new ZoneTimerSchedule(STRIPE_0, executor);
+ ZoneTimerSchedule zoneTimerSchedule1 = new ZoneTimerSchedule(STRIPE_1, executor);
CountDownLatch taskLatch0 = new CountDownLatch(1);
CountDownLatch taskLatch1 = new CountDownLatch(1);
- state0.rescheduleScaleUp(
+ zoneTimerSchedule0.reschedule(
0L,
() -> {
taskLatch0.countDown();
@@ -515,21 +346,20 @@ public class DistributionZonesSchedulersTest {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
- },
- STRIPE_0
+ }
);
// Wait when the first task is started.
assertTrue(taskLatch0.await(3, TimeUnit.SECONDS));
- state1.rescheduleScaleUp(0L, () -> {}, STRIPE_1);
+ zoneTimerSchedule1.reschedule(0L, () -> {});
- assertTrue(waitForCondition(() -> state1.scaleUpTask().isDone(), 3000));
+ assertTrue(waitForCondition(zoneTimerSchedule1::taskIsDone, 3000));
- assertFalse(state0.scaleUpTask().isDone());
+ assertFalse(zoneTimerSchedule0.taskIsDone());
taskLatch1.countDown();
- assertTrue(waitForCondition(() -> state0.scaleUpTask().isDone(), 3000));
+ assertTrue(waitForCondition(zoneTimerSchedule0::taskIsDone, 3000));
}
}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/TopologyAugmentationMapSerializerTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/TopologyAugmentationMapSerializerTest.java
deleted file mode 100644
index 6b20a6f5f03..00000000000
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/TopologyAugmentationMapSerializerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.distributionzones;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.aMapWithSize;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.is;
-
-import java.util.Base64;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentSkipListMap;
-import org.apache.ignite.internal.distributionzones.DistributionZoneManager.Augmentation;
-import org.apache.ignite.internal.versioned.VersionedSerialization;
-import org.junit.jupiter.api.Test;
-
-class TopologyAugmentationMapSerializerTest {
- private static final UUID NODE1_ID = new UUID(0x1234567890ABCDEFL, 0xFEDCBA0987654321L);
- private static final UUID NODE2_ID = new UUID(0xFEDCBA0987654321L, 0x1234567890ABCDEFL);
-
- private final TopologyAugmentationMapSerializer serializer = new TopologyAugmentationMapSerializer();
-
- @Test
- void serializationAndDeserialization() {
- Node node1 = new Node("node1", NODE1_ID);
- Node node2 = new Node("node2", NODE2_ID);
- ConcurrentSkipListMap<Long, Augmentation> originalMap = new ConcurrentSkipListMap<>(Map.of(
- 1000L, new Augmentation(Set.of(node1, node2), true),
- 2000L, new Augmentation(Set.of(node2, node1), false)
- ));
-
- byte[] bytes = VersionedSerialization.toBytes(originalMap, serializer);
- ConcurrentSkipListMap<Long, Augmentation> restoredMap = VersionedSerialization.fromBytes(bytes, serializer);
-
- assertThat(restoredMap, is(aMapWithSize(2)));
-
- Augmentation augmentation1 = restoredMap.get(1000L);
- assertThat(augmentation1.nodes(), containsInAnyOrder(node1, node2));
- assertThat(augmentation1.addition(), is(true));
-
- Augmentation augmentation2 = restoredMap.get(2000L);
- assertThat(augmentation2.nodes(), containsInAnyOrder(node1, node2));
- assertThat(augmentation2.addition(), is(false));
- }
-
- @Test
- void v1CanBeDeserialized() {
- byte[] bytes = Base64.getDecoder().decode("Ae++QwPpBwHvvkMDAe++QwZub2RlMe/Nq5B4VjQSIUNlhwm63P4B775DBm5vZGUyIUNlhwm63P7vzauQeFY0"
- + "EgHRDwHvvkMDAe++QwZub2RlMiFDZYcJutz+782rkHhWNBIB775DBm5vZGUx782rkHhWNBIhQ2WHCbrc/gA=");
- ConcurrentSkipListMap<Long, Augmentation> restoredMap = VersionedSerialization.fromBytes(bytes, serializer);
-
- Node node1 = new Node("node1", NODE1_ID);
- Node node2 = new Node("node2", NODE2_ID);
-
- assertThat(restoredMap, is(aMapWithSize(2)));
-
- Augmentation augmentation1 = restoredMap.get(1000L);
- assertThat(augmentation1.nodes(), containsInAnyOrder(node1, node2));
- assertThat(augmentation1.addition(), is(true));
-
- Augmentation augmentation2 = restoredMap.get(2000L);
- assertThat(augmentation2.nodes(), containsInAnyOrder(node1, node2));
- assertThat(augmentation2.addition(), is(false));
- }
-}
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/DistributionZoneCausalityDataNodesTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/DistributionZoneCausalityDataNodesTest.java
index cf247cc5b4f..84409799317 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/DistributionZoneCausalityDataNodesTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/causalitydatanodes/DistributionZoneCausalityDataNodesTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.distributionzones.causalitydatanodes;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.emptySet;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -30,11 +31,10 @@ import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_DROP;
import static org.apache.ignite.internal.cluster.management.topology.LogicalTopologyImpl.LOGICAL_TOPOLOGY_KEY;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesFromManager;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertValueInStorage;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeDataNodesMap;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeLogicalTopologySet;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesDataNodesPrefix;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyPrefix;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyVersionKey;
@@ -42,6 +42,7 @@ import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUt
import static org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrowsWithCause;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
+import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast;
import static org.apache.ignite.internal.util.ByteUtils.toByteArray;
import static org.apache.ignite.internal.util.CompletableFutures.falseCompletedFuture;
import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
@@ -75,12 +76,17 @@ import org.apache.ignite.internal.cluster.management.topology.api.LogicalNode;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologySnapshot;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologySnapshotSerializer;
import org.apache.ignite.internal.distributionzones.BaseDistributionZoneManagerTest;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.distributionzones.DataNodesHistoryEntry;
import org.apache.ignite.internal.distributionzones.DistributionZoneManager;
import org.apache.ignite.internal.distributionzones.DistributionZonesUtil;
import org.apache.ignite.internal.distributionzones.Node;
import org.apache.ignite.internal.distributionzones.NodeWithAttributes;
import org.apache.ignite.internal.distributionzones.exception.DistributionZoneNotFoundException;
import org.apache.ignite.internal.distributionzones.utils.CatalogAlterZoneEventListener;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
+import org.apache.ignite.internal.lang.ByteArray;
import org.apache.ignite.internal.lang.IgniteBiTuple;
import org.apache.ignite.internal.manager.ComponentContext;
import org.apache.ignite.internal.metastorage.Entry;
@@ -89,6 +95,7 @@ import org.apache.ignite.internal.metastorage.WatchListener;
import org.apache.ignite.internal.metastorage.server.If;
import org.apache.ignite.internal.metastorage.server.raft.MetaStorageWriteHandler;
import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.util.PendingComparableValuesTracker;
import org.apache.ignite.internal.versioned.VersionedSerialization;
import org.apache.ignite.network.ClusterNode;
import org.apache.ignite.network.NetworkAddress;
@@ -183,10 +190,20 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
*/
private final ConcurrentHashMap<Integer, CompletableFuture<Long>> dropZoneRevisions = new ConcurrentHashMap<>();
+ private PendingComparableValuesTracker<Long, Void> revisionsTracker;
+
@BeforeEach
void beforeEach() {
+ revisionsTracker = new PendingComparableValuesTracker<>(0L);
metaStorageManager.registerPrefixWatch(zonesLogicalTopologyPrefix(), createMetastorageTopologyListener());
- metaStorageManager.registerPrefixWatch(zonesDataNodesPrefix(), createMetastorageDataNodesListener());
+ metaStorageManager.registerRevisionUpdateListener(rev -> {
+ revisionsTracker.update(rev, null);
+ return nullCompletedFuture();
+ });
+ metaStorageManager.registerPrefixWatch(
+ new ByteArray(DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX.getBytes(UTF_8)),
+ createMetastorageDataNodesListener()
+ );
addCatalogZoneEventListeners();
@@ -670,8 +687,8 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(zoneId),
- (v) -> DistributionZonesUtil.dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
+ zoneDataNodesHistoryKey(zoneId),
+ DistributionZoneCausalityDataNodesTest::dataNodesFromValue,
ONE_NODE_NAME,
TIMEOUT
);
@@ -724,8 +741,8 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(zoneId),
- (v) -> DistributionZonesUtil.dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
+ zoneDataNodesHistoryKey(zoneId),
+ DistributionZoneCausalityDataNodesTest::dataNodesFromValue,
TWO_NODES_NAMES,
TIMEOUT
);
@@ -960,6 +977,8 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
// Change logical topology. NODE_1 is left.
long topologyRevision3 = removeNodeInLogicalTopologyAndGetRevision(Set.of(NODE_1), twoNodes1);
+ assertTrue(waitForCondition(() -> metaStorageManager.appliedRevision() >= topologyRevision3, 1000));
+
Set<String> dataNodes5 = distributionZoneManager.dataNodes(topologyRevision2, catalogManager.latestCatalogVersion(), defaultZoneId)
.get(TIMEOUT, MILLISECONDS);
assertEquals(THREE_NODES_NAMES, dataNodes5);
@@ -1130,18 +1149,20 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
long topologyRevision;
+ log.info("Test: changing topology.");
+
if (addNode) {
newTopology.add(NODE_2);
// Change logical topology. NODE_2 is added.
topologyRevision = putNodeInLogicalTopologyAndGetRevision(NODE_2, newTopology);
- waitForCondition(() -> metaStorageManager.appliedRevision() >= topologyRevision, TIMEOUT);
+ assertTrue(waitForCondition(() -> metaStorageManager.appliedRevision() >= topologyRevision, TIMEOUT));
} else {
newTopology.remove(NODE_1);
// Change logical topology. NODE_1 is removed.
topologyRevision = removeNodeInLogicalTopologyAndGetRevision(Set.of(NODE_1), newTopology);
- waitForCondition(() -> metaStorageManager.appliedRevision() >= topologyRevision, TIMEOUT);
+ assertTrue(waitForCondition(() -> metaStorageManager.appliedRevision() >= topologyRevision, TIMEOUT));
}
assertEquals(topologyRevision, topologyUpdateRevision.get());
@@ -1153,11 +1174,13 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
// Check that data nodes values are changed in the meta storage after ms invokes updated the meta storage.
checkThatDataNodesIsChangedInMetastorage(expectedDataNodesAfterTimersAreExpired);
- // Check that data nodes values are idempotented.
+ // Check that data nodes values are idempotent.
// Data nodes from the meta storage manager only used to calculate current data nodes because the meta storage are updated and
// topology augmentation maps were cleared.
checkDataNodes(topologyUpdateRevision.get(), expectedDataNodesOnTopologyUpdateEvent);
+ // Above, we have waited for data nodes in meta storage to become the same as in expectedDataNodesAfterTimersAreExpired map,
+ // so we use the applied revision and check the value of data nodes in data nodes manager.
checkDataNodes(metaStorageManager.appliedRevision(), expectedDataNodesAfterTimersAreExpired);
}
@@ -1194,7 +1217,7 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
for (Integer zoneId : zoneIds) {
Set<String> dataNodes = distributionZoneManager.dataNodes(topologyRevision, catalogManager.latestCatalogVersion(), zoneId)
.get(TIMEOUT, MILLISECONDS);
- assertEquals(TWO_NODES_NAMES, dataNodes);
+ assertEquals(TWO_NODES_NAMES, dataNodes, "zoneId=" + zoneId);
}
}
@@ -1218,7 +1241,7 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
return CompletableFuture.runAsync(() -> {
try {
// Check that data nodes values are changed according to scale up and down timers.
- // Data nodes from the meta storage manager and topology augmentation maps are used to calculate current data nodes.
+ // Data nodes from the data nodes manager are used to calculate current data nodes.
checkDataNodes(topologyUpdateRevision.get(), expectedDataNodes);
// Check that data nodes values are not changed in the meta storage.
@@ -1228,7 +1251,11 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
} catch (Exception e) {
fail();
}
- }).thenRun(latch::countDown).thenApply(ignored -> null);
+ })
+ .thenRun(() -> {
+ latch.countDown();
+ log.info("Test: latch counted down.");
+ });
};
}
@@ -1236,14 +1263,20 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
for (Integer zoneId : zoneIds) {
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(zoneId),
- (v) -> DistributionZonesUtil.dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
+ zoneDataNodesHistoryKey(zoneId),
+ DistributionZoneCausalityDataNodesTest::dataNodesFromValue,
TWO_NODES_NAMES,
TIMEOUT
);
}
}
+ private static Set<String> dataNodesFromValue(byte[] value) {
+ DataNodesHistory history = DataNodesHistorySerializer.deserialize(value);
+ DataNodesHistoryEntry latest = history.dataNodesForTimestamp(HybridTimestamp.MAX_VALUE);
+ return DistributionZonesUtil.dataNodes(latest.dataNodes()).stream().map(Node::nodeName).collect(toSet());
+ }
+
/**
* Check current data nodes.
*/
@@ -1252,10 +1285,13 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
Map<Integer, Set<String>> expectedDataNodes
) throws Exception {
for (Map.Entry<Integer, Set<String>> entry : expectedDataNodes.entrySet()) {
+ int zoneId = entry.getKey();
+
assertEquals(
entry.getValue(),
- distributionZoneManager.dataNodes(revision, catalogManager.latestCatalogVersion(), entry.getKey())
- .get(TIMEOUT, MILLISECONDS)
+ distributionZoneManager.dataNodes(revision, catalogManager.latestCatalogVersion(), zoneId)
+ .get(TIMEOUT, MILLISECONDS),
+ "zoneId=" + zoneId
);
}
}
@@ -1266,8 +1302,8 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
for (Map.Entry<Integer, Set<String>> entry : expectedDataNodes.entrySet()) {
assertValueInStorage(
metaStorageManager,
- zoneDataNodesKey(entry.getKey()),
- (v) -> DistributionZonesUtil.dataNodes(deserializeDataNodesMap(v)).stream().map(Node::nodeName).collect(toSet()),
+ zoneDataNodesHistoryKey(entry.getKey()),
+ DistributionZoneCausalityDataNodesTest::dataNodesFromValue,
entry.getValue(),
TIMEOUT
);
@@ -1310,7 +1346,11 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
topology.putNode(node);
- return revisionFut.get(TIMEOUT, MILLISECONDS);
+ long revision = revisionFut.get(TIMEOUT, MILLISECONDS);
+
+ assertThat(revisionsTracker.waitFor(revision), willSucceedFast());
+
+ return revision;
}
/**
@@ -1535,15 +1575,17 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
for (EntryEvent event : evt.entryEvents()) {
Entry e = event.newEntry();
- if (startsWith(e.key(), zoneDataNodesKey().bytes())) {
+ if (startsWith(e.key(), DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES)) {
revision = e.revision();
- zoneId = extractZoneId(e.key(), DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX_BYTES);
+ zoneId = extractZoneId(e.key(), DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX_BYTES);
byte[] dataNodesBytes = e.value();
if (dataNodesBytes != null) {
- newDataNodes = DistributionZonesUtil.dataNodes(deserializeDataNodesMap(dataNodesBytes));
+ DataNodesHistory history = DataNodesHistorySerializer.deserialize(dataNodesBytes);
+ Set<NodeWithAttributes> nwa = history.dataNodesForTimestamp(HybridTimestamp.MAX_VALUE).dataNodes();
+ newDataNodes = nwa.stream().map(NodeWithAttributes::node).collect(toSet());
} else {
newDataNodes = emptySet();
}
@@ -1621,7 +1663,7 @@ public class DistributionZoneCausalityDataNodesTest extends BaseDistributionZone
})).when(metaStorageManager).invoke(argThat(iif -> {
If iif1 = MetaStorageWriteHandler.toIf(iif);
- byte[] zoneDataNodes = zoneDataNodesKey().bytes();
+ byte[] zoneDataNodes = DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX.getBytes(UTF_8);
return iif1.andThen().update().operations().stream().anyMatch(op -> startsWith(toByteArray(op.key()), zoneDataNodes));
}));
diff --git a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineTest.java b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineTest.java
index 53319b49294..c4c41fe7333 100644
--- a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineTest.java
+++ b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/rebalance/DistributionZoneRebalanceEngineTest.java
@@ -25,8 +25,8 @@ import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_
import static org.apache.ignite.internal.catalog.CatalogTestUtils.createTestCatalogManager;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.getDefaultZone;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.getZoneIdStrict;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.toDataNodesMap;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.stablePartAssignmentsKey;
import static org.apache.ignite.internal.hlc.HybridTimestamp.hybridTimestamp;
import static org.apache.ignite.internal.metastorage.server.KeyValueUpdateContext.kvContext;
@@ -54,7 +54,6 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.Serializable;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -68,10 +67,10 @@ import org.apache.ignite.internal.catalog.CatalogManager;
import org.apache.ignite.internal.catalog.commands.ColumnParams;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension;
-import org.apache.ignite.internal.distributionzones.DataNodesMapSerializer;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
import org.apache.ignite.internal.distributionzones.DistributionZoneManager;
import org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil;
-import org.apache.ignite.internal.distributionzones.Node;
import org.apache.ignite.internal.distributionzones.NodeWithAttributes;
import org.apache.ignite.internal.hlc.HybridClock;
import org.apache.ignite.internal.hlc.HybridClockImpl;
@@ -168,7 +167,7 @@ public class DistributionZoneRebalanceEngineTest extends IgniteAbstractTest {
WatchListener watchListener = invocation.getArgument(1);
- if (Arrays.equals(key.bytes(), zoneDataNodesKey().bytes())) {
+ if (new String(key.bytes(), UTF_8).startsWith(DISTRIBUTION_ZONE_DATA_NODES_HISTORY_PREFIX)) {
this.watchListener = watchListener;
}
@@ -538,14 +537,18 @@ public class DistributionZoneRebalanceEngineTest extends IgniteAbstractTest {
byte[] newLogicalTopology;
if (nodes != null) {
- newLogicalTopology = DataNodesMapSerializer.serialize(toDataNodesMap(nodes.stream()
- .map(n -> new Node(n, findNodeIdByConsistentId(n)))
- .collect(toSet())));
+ Set<NodeWithAttributes> nodeWithAttributes = nodes.stream()
+ .map(n -> nodeWithAttributesMap.get(findNodeIdByConsistentId(n)))
+ .collect(toSet());
+
+ DataNodesHistory history = new DataNodesHistory().addHistoryEntry(clock.now(), nodeWithAttributes);
+
+ newLogicalTopology = DataNodesHistorySerializer.serialize(history);
} else {
newLogicalTopology = null;
}
- Entry newEntry = new EntryImpl(zoneDataNodesKey(zoneId).bytes(), newLogicalTopology, rev, clock.now());
+ Entry newEntry = new EntryImpl(zoneDataNodesHistoryKey(zoneId).bytes(), newLogicalTopology, rev, clock.now());
EntryEvent entryEvent = new EntryEvent(null, newEntry);
diff --git a/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java b/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
index 401fbac78b5..844d022fd13 100644
--- a/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
+++ b/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
@@ -17,24 +17,32 @@
package org.apache.ignite.internal.distributionzones;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deserializeDataNodesMap;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.dataNodeHistoryContextFromValues;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.parseDataNodes;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.parseStorageProfiles;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKey;
-import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesHistoryKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownTimerKey;
+import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpTimerKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
import static org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyVersionKey;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -52,11 +60,17 @@ import org.apache.ignite.internal.catalog.commands.DropZoneCommand;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.cluster.management.topology.api.LogicalNode;
+import org.apache.ignite.internal.distributionzones.DataNodesHistory.DataNodesHistorySerializer;
+import org.apache.ignite.internal.distributionzones.DistributionZonesUtil.DataNodesHistoryContext;
+import org.apache.ignite.internal.hlc.HybridTimestamp;
import org.apache.ignite.internal.lang.ByteArray;
+import org.apache.ignite.internal.metastorage.Entry;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
import org.apache.ignite.internal.metastorage.server.KeyValueStorage;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
import org.apache.ignite.internal.util.ByteUtils;
import org.apache.ignite.network.ClusterNode;
+import org.apache.ignite.network.NetworkAddress;
import org.jetbrains.annotations.Nullable;
@@ -297,7 +311,7 @@ public class DistributionZonesTestUtil {
}
/**
- * Asserts data nodes from {@link DistributionZonesUtil#zoneDataNodesKey(int)} in storage with set of LogicalNodes as input.
+ * Asserts data nodes from {@link DistributionZonesUtil#zoneDataNodesHistoryKey(int)} in storage with set of LogicalNodes as input.
*
* @param zoneId Zone id.
* @param clusterNodes Data nodes.
@@ -313,17 +327,15 @@ public class DistributionZonesTestUtil {
? null
: clusterNodes.stream().map(n -> new Node(n.name(), n.id())).collect(toSet());
- assertValueInStorage(
- keyValueStorage,
- zoneDataNodesKey(zoneId).bytes(),
- value -> DistributionZonesUtil.dataNodes(deserializeDataNodesMap(value)),
+ assertDataNodesInStorage(
+ zoneId,
nodes,
- 2000
+ keyValueStorage
);
}
/**
... 601 lines suppressed ...