You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by ta...@apache.org on 2026/08/27 06:46:11 UTC

(ratis) branch master updated: RATIS-2661. Persist bootstrap configuration for group recovery (#1564)

This is an automated email from the ASF dual-hosted git repository.

OneSizeFitsQuorum pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git


The following commit(s) were added to refs/heads/master by this push:
     new 9526195ea RATIS-2661. Persist bootstrap configuration for group recovery (#1564)
9526195ea is described below

commit 9526195ea60b9110f75009983e3b5ef764df378b
Author: Potato <ta...@apache.org>
AuthorDate: Thu Aug 27 14:46:06 2026 +0800

    RATIS-2661. Persist bootstrap configuration for group recovery (#1564)
---
 .../apache/ratis/server/impl/RaftServerImpl.java   |   6 +-
 .../apache/ratis/server/impl/ServerImplUtils.java  |   4 +
 .../org/apache/ratis/server/impl/ServerState.java  |  33 ++++++-
 .../apache/ratis/server/raftlog/LogProtoUtils.java |   7 ++
 .../server/storage/RaftStorageDirectoryImpl.java   |   5 ++
 .../ratis/server/storage/RaftStorageImpl.java      |  52 +++++++++--
 .../ratis/server/impl/GroupManagementBaseTest.java | 100 +++++++++++++++++++++
 .../ratis/server/storage/TestRaftStorage.java      |  37 ++++++++
 8 files changed, 233 insertions(+), 11 deletions(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
index f4e740407..4fb0bf2ed 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
@@ -1938,7 +1938,11 @@ class RaftServerImpl implements RaftServer.Division,
     case CONFIGURATIONENTRY:
       // the reply should have already been set. only need to record
       // the new conf in the metadata file and notify the StateMachine.
-      state.writeRaftConfiguration(next);
+      try {
+        state.writeRaftConfiguration(next);
+      } catch (IOException e) {
+        throw new RaftLogIOException(e);
+      }
       stateMachine.event().notifyConfigurationChanged(next.getTerm(), next.getIndex(),
           next.getConfigurationEntry());
       role.getLeaderState().ifPresent(leader -> leader.checkReady(next));
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java
index 434f98d68..fb1cf1738 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java
@@ -221,6 +221,10 @@ public final class ServerImplUtils {
     return b.build();
   }
 
+  public static RaftConfiguration newRaftConfiguration(List<RaftPeer> conf, List<RaftPeer> listener) {
+    return RaftConfigurationImpl.newBuilder().setConf(conf, listener).build();
+  }
+
   static long effectiveCommitIndex(long leaderCommitIndex, TermIndex followerPrevious, int numAppendEntries) {
     final long previous = followerPrevious != null? followerPrevious.getIndex() : RaftLog.LEAST_VALID_LOG_INDEX;
     return Math.min(leaderCommitIndex, previous + numAppendEntries);
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
index ad3210d63..17762ec2a 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
@@ -129,8 +129,35 @@ class ServerState {
   void initialize(StateMachine stateMachine) throws IOException {
     // initialize raft storage
     final RaftStorageImpl storage = raftStorage.get();
-    // read configuration from the storage
-    Optional.ofNullable(storage.readRaftConfiguration()).ifPresent(this::setRaftConf);
+    final RaftConfigurationImpl initialConf = getRaftConf();
+    final boolean hasInitialConf = !initialConf.getCurrentPeers().isEmpty()
+        || !initialConf.getCurrentPeers(RaftPeerRole.LISTENER).isEmpty();
+
+    // The latest applied Raft configuration always takes precedence.  The bootstrap
+    // configuration is only a durable seed until the first configuration entry is applied.
+    final RaftConfiguration persistedConf = storage.readRaftConfiguration();
+    if (persistedConf != null) {
+      storage.deleteBootstrapConfiguration();
+      setRaftConf(persistedConf);
+    } else {
+      final RaftConfiguration bootstrapConf = storage.readBootstrapConfiguration();
+      if (bootstrapConf != null) {
+        if (!hasInitialConf) {
+          setRaftConf(bootstrapConf);
+        } else if (!((RaftConfigurationImpl) bootstrapConf).hasNoChange(
+            initialConf.getCurrentPeers(), initialConf.getCurrentPeers(RaftPeerRole.LISTENER))) {
+          throw new IOException("Conflicting bootstrap configuration for " + getMemberId()
+              + ": stored=" + bootstrapConf + ", supplied=" + initialConf);
+        } else {
+          // Preserve the supplied peer metadata, such as updated addresses, while keeping
+          // the voting membership and priorities unchanged.
+          storage.writeBootstrapConfiguration(initialConf);
+        }
+      } else if (hasInitialConf) {
+        // Persist before start() can acknowledge the group-add request.
+        storage.writeBootstrapConfiguration(initialConf);
+      }
+    }
 
     stateMachine.initialize(server.getRaftServer(), getMemberId().getGroupId(), storage);
 
@@ -155,7 +182,7 @@ class ServerState {
     return index >= 0 ? index : RaftLog.INVALID_LOG_INDEX;
   }
 
-  void writeRaftConfiguration(LogEntryProto conf) {
+  void writeRaftConfiguration(LogEntryProto conf) throws IOException {
     getStorage().writeRaftConfiguration(conf);
   }
 
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java
index 3705c3bd4..ef4d0b7ed 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java
@@ -240,4 +240,11 @@ public final class LogProtoUtils {
     final List<RaftPeer> oldListener = ProtoUtils.toRaftPeers(proto.getOldListenersList());
     return ServerImplUtils.newRaftConfiguration(conf, listener, entry.getIndex(), oldConf, oldListener);
   }
+
+  /** Convert a bootstrap configuration, which has no corresponding log entry. */
+  public static RaftConfiguration toRaftConfiguration(RaftConfigurationProto proto) {
+    final List<RaftPeer> conf = ProtoUtils.toRaftPeers(proto.getPeersList());
+    final List<RaftPeer> listener = ProtoUtils.toRaftPeers(proto.getListenersList());
+    return ServerImplUtils.newRaftConfiguration(conf, listener);
+  }
 }
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java
index 119f7922d..e1f60013e 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java
@@ -40,6 +40,7 @@ class RaftStorageDirectoryImpl implements RaftStorageDirectory {
   private static final String IN_USE_LOCK_NAME = "in_use.lock";
   private static final String META_FILE_NAME = "raft-meta";
   private static final String CONF_EXTENSION = ".conf";
+  private static final String BOOTSTRAP_CONF_EXTENSION = ".bootstrap";
   private static final String JVM_NAME = ManagementFactory.getRuntimeMXBean().getName();
 
   enum StorageState {
@@ -106,6 +107,10 @@ class RaftStorageDirectoryImpl implements RaftStorageDirectory {
     return new File(getCurrentDir(), META_FILE_NAME + CONF_EXTENSION);
   }
 
+  File getBootstrapConfFile() {
+    return new File(getCurrentDir(), META_FILE_NAME + BOOTSTRAP_CONF_EXTENSION);
+  }
+
   /**
    * Check to see if current/ directory is empty.
    */
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
index ce809cad8..33dc333bb 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
@@ -21,6 +21,8 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ratis.proto.RaftProtos.LogEntryProto;
+import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto;
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
 import org.apache.ratis.server.RaftConfiguration;
 import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
 import org.apache.ratis.server.raftlog.LogProtoUtils;
@@ -141,16 +143,15 @@ public class RaftStorageImpl implements RaftStorage {
     return metaFile.get();
   }
 
-  public void writeRaftConfiguration(LogEntryProto conf) {
+  public void writeRaftConfiguration(LogEntryProto conf) throws IOException {
     File confFile = storageDir.getMetaConfFile();
     try (OutputStream fio = new AtomicFileOutputStream(confFile)) {
       conf.writeTo(fio);
-    } catch (Exception e) {
-      LOG.error("Failed writing configuration to file:" + confFile, e);
     }
+    deleteBootstrapConfiguration();
   }
 
-  public RaftConfiguration readRaftConfiguration() {
+  public RaftConfiguration readRaftConfiguration() throws IOException {
     File confFile = storageDir.getMetaConfFile();
     if (!confFile.exists()) {
       return null;
@@ -158,13 +159,50 @@ public class RaftStorageImpl implements RaftStorage {
       try (InputStream fio = FileUtils.newInputStream(confFile)) {
         LogEntryProto confProto = LogEntryProto.newBuilder().mergeFrom(fio).build();
         return LogProtoUtils.toRaftConfiguration(confProto);
-      } catch (Exception e) {
-        LOG.error("Failed reading configuration from file:" + confFile, e);
-        return null;
       }
     }
   }
 
+  public void writeBootstrapConfiguration(RaftConfiguration conf) throws IOException {
+    if (conf.getCurrentPeers().isEmpty()
+        && conf.getCurrentPeers(RaftPeerRole.LISTENER).isEmpty()) {
+      throw new IOException("Refusing to persist an empty bootstrap configuration");
+    }
+    if (!conf.getPreviousPeers().isEmpty()
+        || !conf.getPreviousPeers(RaftPeerRole.LISTENER).isEmpty()) {
+      throw new IOException("Refusing to persist a transitional bootstrap configuration: " + conf);
+    }
+    final File confFile = storageDir.getBootstrapConfFile();
+    try (OutputStream out = new AtomicFileOutputStream(confFile)) {
+      LogProtoUtils.toRaftConfigurationProtoBuilder(conf).build().writeTo(out);
+    }
+  }
+
+  public RaftConfiguration readBootstrapConfiguration() throws IOException {
+    final File confFile = storageDir.getBootstrapConfFile();
+    if (!confFile.exists()) {
+      return null;
+    }
+    try (InputStream in = FileUtils.newInputStream(confFile)) {
+      final RaftConfigurationProto proto = RaftConfigurationProto.newBuilder().mergeFrom(in).build();
+      if (proto.getPeersCount() == 0 && proto.getListenersCount() == 0) {
+        throw new IOException("Invalid empty bootstrap configuration in " + confFile);
+      }
+      if (proto.getOldPeersCount() != 0 || proto.getOldListenersCount() != 0) {
+        throw new IOException("Invalid transitional bootstrap configuration in " + confFile);
+      }
+      try {
+        return LogProtoUtils.toRaftConfiguration(proto);
+      } catch (RuntimeException e) {
+        throw new IOException("Invalid bootstrap configuration in " + confFile, e);
+      }
+    }
+  }
+
+  public void deleteBootstrapConfiguration() throws IOException {
+    FileUtils.deleteIfExists(storageDir.getBootstrapConfFile());
+  }
+
   @Override
   public String toString() {
     return JavaUtils.getClassSimpleName(getClass()) + ":" + getStorageDir();
diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
index a86ab2a51..12f902811 100644
--- a/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
+++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
@@ -33,6 +33,7 @@ import org.apache.ratis.server.RaftServer;
 import org.apache.ratis.server.RaftServerConfigKeys;
 import org.apache.ratis.util.FileUtils;
 import org.apache.ratis.util.JavaUtils;
+import org.apache.ratis.util.LifeCycle;
 import org.apache.ratis.util.Slf4jUtils;
 import org.apache.ratis.util.TimeDuration;
 import org.apache.ratis.util.function.CheckedBiConsumer;
@@ -45,10 +46,12 @@ import org.slf4j.event.Level;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.Random;
@@ -216,6 +219,103 @@ public abstract class GroupManagementBaseTest extends BaseTest {
     cluster.shutdown();
   }
 
+  @Test
+  public void testRecoverGroupBeforeInitialConfigurationEntry() throws Exception {
+    final MiniRaftCluster cluster = getCluster(0);
+    final BlockRequestHandlingInjection injection = BlockRequestHandlingInjection.getInstance();
+
+    try {
+      // Start three servers without an initial group.  The group will be added dynamically below.
+      final List<RaftPeerId> ids = Arrays.stream(MiniRaftCluster.generateIds(3, 0))
+          .map(RaftPeerId::valueOf).collect(Collectors.toList());
+      ids.forEach(id -> cluster.putNewServer(id, null, true));
+      cluster.start();
+
+      final List<RaftPeer> peers = cluster.getPeers();
+      final RaftGroup group = RaftGroup.valueOf(RaftGroupId.randomId(), peers);
+
+      // Prevent every candidate from receiving votes, so no leader can append the initial
+      // configuration entry before the servers are restarted.
+      peers.forEach(peer -> injection.blockRequestor(peer.getId().toString()));
+      try (RaftClient client = cluster.createClient(group)) {
+        for (RaftPeer peer : peers) {
+          client.getGroupManagementApi(peer.getId()).add(group);
+        }
+      }
+
+      cluster.getTimeoutMax().sleep();
+      for (RaftPeer peer : peers) {
+        final RaftServer.Division division = cluster.getDivision(peer.getId(), group.getGroupId());
+        Assertions.assertEquals(LifeCycle.State.RUNNING, division.getInfo().getLifeCycleState());
+        Assertions.assertNull(division.getRaftLog().getLastEntryTermIndex());
+        Assertions.assertFalse(division.getGroup().getPeers().isEmpty());
+        Assertions.assertTrue(new File(division.getRaftStorage().getStorageDir().getCurrentDir(),
+            "raft-meta.bootstrap").isFile());
+      }
+
+      // Restart through directory scanning: passing a null group makes recovery reconstruct
+      // the division from its directory name (group ID) only.
+      for (RaftPeer peer : peers) {
+        cluster.restartServer(peer.getId(), null, false);
+      }
+      injection.unblockAll();
+
+      cluster.getTimeoutMax().sleep();
+      for (RaftPeer peer : peers) {
+        final RaftServer.Division division = cluster.getDivision(peer.getId(), group.getGroupId());
+        Assertions.assertIterableEquals(peers, division.getGroup().getPeers());
+        Assertions.assertEquals(LifeCycle.State.RUNNING, division.getInfo().getLifeCycleState());
+      }
+      Assertions.assertNotNull(RaftTestUtil.waitForLeader(cluster, group.getGroupId()));
+      JavaUtils.attempt(() -> {
+        for (RaftPeer peer : peers) {
+          final RaftServer.Division division = cluster.getDivision(peer.getId(), group.getGroupId());
+          Assertions.assertFalse(new File(division.getRaftStorage().getStorageDir().getCurrentDir(),
+              "raft-meta.bootstrap").exists());
+        }
+      }, 10, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS),
+          "wait for bootstrap configuration cleanup", LOG);
+    } finally {
+      injection.unblockAll();
+      cluster.shutdown();
+    }
+  }
+
+  @Test
+  public void testRejectConflictingBootstrapConfiguration() throws Exception {
+    final MiniRaftCluster cluster = getCluster(0);
+    try {
+      final List<RaftPeerId> ids = Arrays.stream(MiniRaftCluster.generateIds(3, 0))
+          .map(RaftPeerId::valueOf).collect(Collectors.toList());
+      ids.forEach(id -> cluster.putNewServer(id, null, true));
+      cluster.start();
+
+      final List<RaftPeer> peers = cluster.getPeers();
+      final RaftGroup group = RaftGroup.valueOf(RaftGroupId.randomId(), peers);
+      final RaftPeer target = peers.get(0);
+      try (RaftClient client = cluster.createClient(group)) {
+        client.getGroupManagementApi(target.getId()).add(group);
+      }
+
+      final RaftServer.Division division = cluster.getDivision(target.getId(), group.getGroupId());
+      Assertions.assertNull(division.getRaftLog().getLastEntryTermIndex());
+
+      final List<RaftPeer> conflictingPeers = new ArrayList<>(peers.subList(0, 2));
+      conflictingPeers.add(RaftPeer.newBuilder()
+          .setId("conflicting-peer")
+          .setAddress("localhost:12345")
+          .build());
+      final RaftGroup conflictingGroup = RaftGroup.valueOf(group.getGroupId(), conflictingPeers);
+      final CompletionException exception = Assertions.assertThrows(CompletionException.class,
+          () -> cluster.restartServer(target.getId(), conflictingGroup, false));
+      final Throwable cause = JavaUtils.unwrapCompletionException(exception);
+      Assertions.assertInstanceOf(IOException.class, cause);
+      Assertions.assertTrue(cause.toString().contains("Conflicting bootstrap configuration"), cause::toString);
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
   @Test
   public void testMultiGroup5Nodes() throws Exception {
     final int[] idIndex = {3, 4, 5};
diff --git a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
index 093e9add7..ca35712e7 100644
--- a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
+++ b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
@@ -24,9 +24,15 @@ import static org.apache.ratis.util.MD5FileUtil.MD5_SUFFIX;
 
 import org.apache.ratis.BaseTest;
 import org.apache.ratis.RaftTestUtil;
+import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto;
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.protocol.RaftPeer;
 import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.server.RaftConfiguration;
 import org.apache.ratis.server.RaftServerConfigKeys;
+import org.apache.ratis.server.impl.ServerImplUtils;
 import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.server.raftlog.LogProtoUtils;
 import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
 import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
 import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
@@ -173,6 +179,37 @@ public class TestRaftStorage extends BaseTest {
     storage.close();
   }
 
+  @Test
+  public void testBootstrapConfiguration() throws Exception {
+    final RaftPeer follower = RaftPeer.newBuilder()
+        .setId("follower")
+        .setAddress("localhost:1001")
+        .setPriority(1)
+        .build();
+    final RaftPeer listener = RaftPeer.newBuilder()
+        .setId("listener")
+        .setAddress("localhost:1002")
+        .setStartupRole(RaftPeerRole.LISTENER)
+        .build();
+    final RaftConfiguration bootstrap = ServerImplUtils.newRaftConfiguration(
+        Collections.singletonList(follower), Collections.singletonList(listener));
+    final RaftConfigurationProto expected = LogProtoUtils.toRaftConfigurationProtoBuilder(bootstrap).build();
+
+    try (RaftStorageImpl storage = formatRaftStorage(storageDir)) {
+      storage.writeBootstrapConfiguration(bootstrap);
+      final RaftConfiguration recovered = storage.readBootstrapConfiguration();
+      Assertions.assertEquals(expected,
+          LogProtoUtils.toRaftConfigurationProtoBuilder(recovered).build());
+
+      storage.writeRaftConfiguration(LogProtoUtils.toLogEntryProto(bootstrap, 1L, 0L));
+      Assertions.assertNull(storage.readBootstrapConfiguration());
+      Assertions.assertEquals(0L, storage.readRaftConfiguration().getLogEntryIndex());
+
+      Files.write(storage.getStorageDir().getBootstrapConfFile().toPath(), new byte[0]);
+      Assertions.assertThrows(IOException.class, storage::readBootstrapConfiguration);
+    }
+  }
+
   /**
    * check if RaftStorage deletes tmp metafile when startup
    */