You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by tk...@apache.org on 2023/11/15 14:32:38 UTC

(ignite-3) branch main updated: IGNITE-20866 Resolved a race condition issue in the cluster configuration initialization process (#2844)

This is an automated email from the ASF dual-hosted git repository.

tkalkirill pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git


The following commit(s) were added to refs/heads/main by this push:
     new 75d50f1c7d IGNITE-20866 Resolved a race condition issue in the cluster configuration initialization process (#2844)
75d50f1c7d is described below

commit 75d50f1c7d447ef7f71bed7dc6502b8b392bf11a
Author: Ivan Gagarkin <ga...@gmail.com>
AuthorDate: Wed Nov 15 21:32:32 2023 +0700

    IGNITE-20866 Resolved a race condition issue in the cluster configuration initialization process (#2844)
    
    * IGNITE-20866 Resolved a race condition issue in the cluster configuration initialization process. Previously, the configuration was applied locally when the meta storage revision was 0, leading to potential race conditions and the configuration being ignored if the revision changed.
    
    Changes made:
    - Altered the initialization logic to first check for the local revision of the configuration key.
    - If the local revision is absent, the system now falls back to the initial configuration.
    
    This fix ensures more reliable and consistent application of the cluster configuration, particularly during initialization phases where the meta storage revision is susceptible to change.
---
 .../configuration/ConfigurationChanger.java        | 15 ++++----
 .../storage/ConfigurationStorage.java              |  3 ++
 .../storage/TestConfigurationStorage.java          |  5 +++
 .../org/apache/ignite/internal/app/IgniteImpl.java | 42 +++++++++++----------
 .../storage/DistributedConfigurationStorage.java   | 44 +++++++---------------
 .../storage/LocalConfigurationStorage.java         |  5 +++
 .../storage/LocalFileConfigurationStorage.java     |  5 +++
 7 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationChanger.java b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationChanger.java
index e7c5f42241..1c6a65a31c 100644
--- a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationChanger.java
+++ b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/ConfigurationChanger.java
@@ -97,8 +97,8 @@ public abstract class ConfigurationChanger implements DynamicConfigurationChange
     private volatile StorageRoots storageRoots;
 
     /**
-     * Initial configuration. This configuration will be used to initialize the configuration if the storage is empty. If the storage is not
-     * empty, this configuration will be ignored.
+     * Initial configuration. This configuration will be used to initialize the configuration if the revision of the storage is {@code 0}.
+     * If the revision of the storage is non-zero, this configuration will be ignored.
      */
     private volatile ConfigurationSource initialConfiguration = ConfigurationUtil.EMPTY_CFG_SRC;
 
@@ -255,7 +255,7 @@ public abstract class ConfigurationChanger implements DynamicConfigurationChange
         }
 
         Map<String, ? extends Serializable> storageValues = data.values();
-        long version = data.changeId();
+        long revision = data.changeId();
 
         SuperRoot superRoot = new SuperRoot(rootCreator());
 
@@ -273,12 +273,13 @@ public abstract class ConfigurationChanger implements DynamicConfigurationChange
             superRoot.addRoot(rootKey, rootNode);
         }
 
+        // Create a copy of the super root, excluding the initial configuration, for saving with the defaults.
         SuperRoot superRootNoDefaults = superRoot.copy();
 
         addDefaults(superRoot);
 
         // Fill the configuration with the initial configuration.
-        if (version == 0) {
+        if (revision == 0) {
             initialConfiguration.descend(superRoot);
         }
 
@@ -321,9 +322,9 @@ public abstract class ConfigurationChanger implements DynamicConfigurationChange
     }
 
     /**
-     * Sets {@link #initialConfiguration}. This configuration will be used to initialize the configuration if the storage is empty. If the
-     * storage is not empty, this configuration will be ignored. This method must be called before {@link #start()}. If the method is not
-     * called, the initial configuration will be empty.
+     * Sets {@link #initialConfiguration}. This configuration will be used to initialize the configuration if the revision of the storage is
+     * {@code 0}. If the revision of the storage is non-zero, this configuration will be ignored. his method must be called before
+     * {@link #start()}. If the method is not called, the initial configuration will be empty.
      *
      * @param configurationSource the configuration source to initialize with.
      */
diff --git a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/storage/ConfigurationStorage.java b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/storage/ConfigurationStorage.java
index d9663eb284..ac2bba580c 100644
--- a/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/storage/ConfigurationStorage.java
+++ b/modules/configuration/src/main/java/org/apache/ignite/internal/configuration/storage/ConfigurationStorage.java
@@ -79,6 +79,9 @@ public interface ConfigurationStorage extends ManuallyCloseable {
      */
     CompletableFuture<Long> lastRevision();
 
+    /** Returns a future that will be completed with the latest revision of the configuration storage. */
+    CompletableFuture<Long> localRevision();
+
     /**
      * Closes the storage.
      */
diff --git a/modules/configuration/src/testFixtures/java/org/apache/ignite/internal/configuration/storage/TestConfigurationStorage.java b/modules/configuration/src/testFixtures/java/org/apache/ignite/internal/configuration/storage/TestConfigurationStorage.java
index cb6733217a..3ea658c699 100644
--- a/modules/configuration/src/testFixtures/java/org/apache/ignite/internal/configuration/storage/TestConfigurationStorage.java
+++ b/modules/configuration/src/testFixtures/java/org/apache/ignite/internal/configuration/storage/TestConfigurationStorage.java
@@ -169,6 +169,11 @@ public class TestConfigurationStorage implements ConfigurationStorage {
         return CompletableFuture.completedFuture(version);
     }
 
+    @Override
+    public CompletableFuture<Long> localRevision() {
+        return lastRevision();
+    }
+
     /**
      * Increase the current revision of the storage.
      *
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
index b9b817efc4..a7f6d4c006 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.app;
 
+import static java.util.concurrent.CompletableFuture.completedFuture;
+
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import java.io.IOException;
@@ -796,14 +798,7 @@ public class IgniteImpl implements Ignite {
 
                         return metaStorageMgr.recoveryFinishedFuture();
                     }, startupExecutor)
-                    .thenComposeAsync(revision -> {
-                        // If the revision is greater than 0, then the configuration has already been initialized.
-                        if (revision > 0) {
-                            return CompletableFuture.completedFuture(null);
-                        } else {
-                            return initializeClusterConfiguration(startupExecutor);
-                        }
-                    }, startupExecutor)
+                    .thenComposeAsync(unused -> initializeClusterConfiguration(startupExecutor), startupExecutor)
                     .thenRunAsync(() -> {
                         LOG.info("MetaStorage started, starting the remaining components");
 
@@ -954,7 +949,7 @@ public class IgniteImpl implements Ignite {
     /** {@inheritDoc} */
     @Override
     public CompletableFuture<Collection<ClusterNode>> clusterNodesAsync() {
-        return CompletableFuture.completedFuture(clusterNodes());
+        return completedFuture(clusterNodes());
     }
 
     /**
@@ -1043,18 +1038,27 @@ public class IgniteImpl implements Ignite {
     }
 
     /**
-     * Initializes the cluster configuration with the specified user-provided configuration upon cluster initialization.
+     * Checks if the local revision is {@code 0} and initializes the cluster configuration with the specified user-provided configuration
+     * upon cluster initialization. If the local revision is not {@code 0}, does nothing.
      */
     private CompletableFuture<Void> initializeClusterConfiguration(ExecutorService startupExecutor) {
-        return cmgMgr.initialClusterConfigurationFuture().thenAcceptAsync(cfg -> {
-            if (cfg == null) {
-                return;
-            }
-
-            Config config = ConfigFactory.parseString(cfg);
-            ConfigurationSource hoconSource = HoconConverter.hoconSource(config.root());
-            clusterCfgMgr.configurationRegistry().initializeConfigurationWith(hoconSource);
-        }, startupExecutor);
+        return cfgStorage.localRevision()
+                .thenComposeAsync(appliedRevision -> {
+                    if (appliedRevision != 0) {
+                        return completedFuture(null);
+                    } else {
+                        return cmgMgr.initialClusterConfigurationFuture()
+                                .thenAcceptAsync(initialConfigHocon -> {
+                                    if (initialConfigHocon == null) {
+                                        return;
+                                    }
+
+                                    Config config = ConfigFactory.parseString(initialConfigHocon);
+                                    ConfigurationSource hoconSource = HoconConverter.hoconSource(config.root());
+                                    clusterCfgMgr.configurationRegistry().initializeConfigurationWith(hoconSource);
+                                }, startupExecutor);
+                    }
+                }, startupExecutor);
     }
 
     /**
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/DistributedConfigurationStorage.java b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/DistributedConfigurationStorage.java
index 9cc4f71bea..8b0a2a8b89 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/DistributedConfigurationStorage.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/DistributedConfigurationStorage.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.configuration.storage;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.ignite.internal.metastorage.dsl.Conditions.notExists;
-import static org.apache.ignite.internal.metastorage.dsl.Conditions.or;
 import static org.apache.ignite.internal.metastorage.dsl.Conditions.revision;
 
 import java.io.Serializable;
@@ -46,7 +45,6 @@ import org.apache.ignite.internal.metastorage.MetaStorageManager;
 import org.apache.ignite.internal.metastorage.WatchEvent;
 import org.apache.ignite.internal.metastorage.WatchListener;
 import org.apache.ignite.internal.metastorage.dsl.Condition;
-import org.apache.ignite.internal.metastorage.dsl.ConditionType;
 import org.apache.ignite.internal.metastorage.dsl.Operation;
 import org.apache.ignite.internal.metastorage.dsl.Operations;
 import org.apache.ignite.internal.thread.NamedThreadFactory;
@@ -86,14 +84,6 @@ public class DistributedConfigurationStorage implements ConfigurationStorage {
      * possible that {@code changeId} is already updated but notifications are not yet handled, thus revision is valid but not applied. This
      * is fine.
      *
-     * <p>Given that {@link #MASTER_KEY} is updated on every configuration change, one could assume that {@code changeId} matches the
-     * revision of {@link #MASTER_KEY}.
-     *
-     * <p>This is true for all cases except for node restart. We use latest values after restart, so MetaStorage's local revision is used
-     * instead. This fact has very important side effect: it's no longer possible to use {@link ConditionType#REV_EQUAL} on
-     * {@link #MASTER_KEY} in {@link DistributedConfigurationStorage#write(Map, long)}. {@link ConditionType#REV_LESS_OR_EQUAL} must be
-     * used instead.
-     *
      * @see #MASTER_KEY
      * @see #write(Map, long)
      */
@@ -191,13 +181,14 @@ public class DistributedConfigurationStorage implements ConfigurationStorage {
         return registerFuture(future);
     }
 
-    private Data readDataOnRecovery0(long cfgRevision) {
-        var data = new HashMap<String, Serializable>();
+    private Data readDataOnRecovery0(long metaStorageRevision) {
+        Map<String, Serializable> data = new HashMap<>();
+        long cfgRevision = 0;
 
         byte[] masterKey = MASTER_KEY.bytes();
         boolean sawMasterKey = false;
 
-        try (Cursor<Entry> cursor = metaStorageMgr.prefixLocally(DST_KEYS_START_RANGE, cfgRevision)) {
+        try (Cursor<Entry> cursor = metaStorageMgr.prefixLocally(DST_KEYS_START_RANGE, metaStorageRevision)) {
             for (Entry entry : cursor) {
                 if (entry.tombstone()) {
                     continue;
@@ -211,6 +202,7 @@ public class DistributedConfigurationStorage implements ConfigurationStorage {
 
                 if (!sawMasterKey && Arrays.equals(masterKey, key)) {
                     sawMasterKey = true;
+                    cfgRevision = entry.revision();
 
                     continue;
                 }
@@ -263,24 +255,10 @@ public class DistributedConfigurationStorage implements ConfigurationStorage {
 
         // Condition for a valid MetaStorage data update. Several possibilities here:
         //  - First update ever, MASTER_KEY property must be absent from MetaStorage.
-        //  - Current node has already performed some updates or received them from MetaStorage watch listener. In this
-        //    case "curChangeId" must match the MASTER_KEY revision exactly.
-        //  - Current node has been restarted and received updates from MetaStorage watch listeners after that. Same as
-        //    above, "curChangeId" must match the MASTER_KEY revision exactly.
-        //  - Current node has been restarted and have not received any updates from MetaStorage watch listeners yet.
-        //    In this case "curChangeId" matches MetaStorage's local revision, which may or may not match the MASTER_KEY revision. Two
-        //    options here:
-        //     - MASTER_KEY is missing in local MetaStorage copy. This means that current node have not performed nor
-        //       observed any configuration changes. Valid condition is "MASTER_KEY does not exist".
-        //     - MASTER_KEY is present in local MetaStorage copy. The MASTER_KEY revision is unknown but is less than or
-        //       equal to MetaStorage's local revision. Obviously, there have been no updates from the future yet. It's also guaranteed
-        //       that the next received configuration update will have the MASTER_KEY revision strictly greater than
-        //       current MetaStorage's local revision. This allows to conclude that "MASTER_KEY revision <= curChangeId" is a valid
-        //       condition for update.
-        // Joining all of the above, it's concluded that the following condition must be used:
-        Condition condition = curChangeId == 0L
+        //  - Otherwise, MASTER_KEY property must be present in MetaStorage and its revision must match "curChangeId".
+        Condition condition = curChangeId == 0
                 ? notExists(MASTER_KEY)
-                : or(notExists(MASTER_KEY), revision(MASTER_KEY).le(curChangeId));
+                : revision(MASTER_KEY).eq(curChangeId);
 
         return metaStorageMgr.invoke(condition, operations, Set.of(Operations.noop()));
     }
@@ -345,6 +323,12 @@ public class DistributedConfigurationStorage implements ConfigurationStorage {
         return metaStorageMgr.get(MASTER_KEY).thenApply(Entry::revision);
     }
 
+    @Override
+    public CompletableFuture<Long> localRevision() {
+        return metaStorageMgr.recoveryFinishedFuture()
+                .thenApply(rev -> metaStorageMgr.getLocally(MASTER_KEY, rev).revision());
+    }
+
     private <T> CompletableFuture<T> registerFuture(CompletableFuture<T> future) {
         futureTracker.registerFuture(future);
 
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalConfigurationStorage.java b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalConfigurationStorage.java
index 9dbfd81546..bc478aab4a 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalConfigurationStorage.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalConfigurationStorage.java
@@ -224,6 +224,11 @@ public class LocalConfigurationStorage implements ConfigurationStorage {
                 .thenApply(entry -> entry == null ? 0 : (Long) fromBytes(entry.value()));
     }
 
+    @Override
+    public CompletableFuture<Long> localRevision() {
+        return lastRevision();
+    }
+
     private <T> CompletableFuture<T> registerFuture(CompletableFuture<T> future) {
         futureTracker.registerFuture(future);
 
diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
index 742745c4cb..a3b1ac4b54 100644
--- a/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
+++ b/modules/runner/src/main/java/org/apache/ignite/internal/configuration/storage/LocalFileConfigurationStorage.java
@@ -219,6 +219,11 @@ public class LocalFileConfigurationStorage implements ConfigurationStorage {
         return CompletableFuture.completedFuture(lastRevision);
     }
 
+    @Override
+    public CompletableFuture<Long> localRevision() {
+        return lastRevision();
+    }
+
     @Override
     public void close() {
         futureTracker.cancelInFlightFutures();