You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by ta...@apache.org on 2026/08/27 06:46:11 UTC
(ratis) branch master updated: RATIS-2661. Persist bootstrap configuration for group recovery (#1564)
This is an automated email from the ASF dual-hosted git repository.
OneSizeFitsQuorum pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 9526195ea RATIS-2661. Persist bootstrap configuration for group recovery (#1564)
9526195ea is described below
commit 9526195ea60b9110f75009983e3b5ef764df378b
Author: Potato <ta...@apache.org>
AuthorDate: Thu Aug 27 14:46:06 2026 +0800
RATIS-2661. Persist bootstrap configuration for group recovery (#1564)
---
.../apache/ratis/server/impl/RaftServerImpl.java | 6 +-
.../apache/ratis/server/impl/ServerImplUtils.java | 4 +
.../org/apache/ratis/server/impl/ServerState.java | 33 ++++++-
.../apache/ratis/server/raftlog/LogProtoUtils.java | 7 ++
.../server/storage/RaftStorageDirectoryImpl.java | 5 ++
.../ratis/server/storage/RaftStorageImpl.java | 52 +++++++++--
.../ratis/server/impl/GroupManagementBaseTest.java | 100 +++++++++++++++++++++
.../ratis/server/storage/TestRaftStorage.java | 37 ++++++++
8 files changed, 233 insertions(+), 11 deletions(-)
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
index f4e740407..4fb0bf2ed 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
@@ -1938,7 +1938,11 @@ class RaftServerImpl implements RaftServer.Division,
case CONFIGURATIONENTRY:
// the reply should have already been set. only need to record
// the new conf in the metadata file and notify the StateMachine.
- state.writeRaftConfiguration(next);
+ try {
+ state.writeRaftConfiguration(next);
+ } catch (IOException e) {
+ throw new RaftLogIOException(e);
+ }
stateMachine.event().notifyConfigurationChanged(next.getTerm(), next.getIndex(),
next.getConfigurationEntry());
role.getLeaderState().ifPresent(leader -> leader.checkReady(next));
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java
index 434f98d68..fb1cf1738 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java
@@ -221,6 +221,10 @@ public final class ServerImplUtils {
return b.build();
}
+ public static RaftConfiguration newRaftConfiguration(List<RaftPeer> conf, List<RaftPeer> listener) {
+ return RaftConfigurationImpl.newBuilder().setConf(conf, listener).build();
+ }
+
static long effectiveCommitIndex(long leaderCommitIndex, TermIndex followerPrevious, int numAppendEntries) {
final long previous = followerPrevious != null? followerPrevious.getIndex() : RaftLog.LEAST_VALID_LOG_INDEX;
return Math.min(leaderCommitIndex, previous + numAppendEntries);
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
index ad3210d63..17762ec2a 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java
@@ -129,8 +129,35 @@ class ServerState {
void initialize(StateMachine stateMachine) throws IOException {
// initialize raft storage
final RaftStorageImpl storage = raftStorage.get();
- // read configuration from the storage
- Optional.ofNullable(storage.readRaftConfiguration()).ifPresent(this::setRaftConf);
+ final RaftConfigurationImpl initialConf = getRaftConf();
+ final boolean hasInitialConf = !initialConf.getCurrentPeers().isEmpty()
+ || !initialConf.getCurrentPeers(RaftPeerRole.LISTENER).isEmpty();
+
+ // The latest applied Raft configuration always takes precedence. The bootstrap
+ // configuration is only a durable seed until the first configuration entry is applied.
+ final RaftConfiguration persistedConf = storage.readRaftConfiguration();
+ if (persistedConf != null) {
+ storage.deleteBootstrapConfiguration();
+ setRaftConf(persistedConf);
+ } else {
+ final RaftConfiguration bootstrapConf = storage.readBootstrapConfiguration();
+ if (bootstrapConf != null) {
+ if (!hasInitialConf) {
+ setRaftConf(bootstrapConf);
+ } else if (!((RaftConfigurationImpl) bootstrapConf).hasNoChange(
+ initialConf.getCurrentPeers(), initialConf.getCurrentPeers(RaftPeerRole.LISTENER))) {
+ throw new IOException("Conflicting bootstrap configuration for " + getMemberId()
+ + ": stored=" + bootstrapConf + ", supplied=" + initialConf);
+ } else {
+ // Preserve the supplied peer metadata, such as updated addresses, while keeping
+ // the voting membership and priorities unchanged.
+ storage.writeBootstrapConfiguration(initialConf);
+ }
+ } else if (hasInitialConf) {
+ // Persist before start() can acknowledge the group-add request.
+ storage.writeBootstrapConfiguration(initialConf);
+ }
+ }
stateMachine.initialize(server.getRaftServer(), getMemberId().getGroupId(), storage);
@@ -155,7 +182,7 @@ class ServerState {
return index >= 0 ? index : RaftLog.INVALID_LOG_INDEX;
}
- void writeRaftConfiguration(LogEntryProto conf) {
+ void writeRaftConfiguration(LogEntryProto conf) throws IOException {
getStorage().writeRaftConfiguration(conf);
}
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java
index 3705c3bd4..ef4d0b7ed 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java
@@ -240,4 +240,11 @@ public final class LogProtoUtils {
final List<RaftPeer> oldListener = ProtoUtils.toRaftPeers(proto.getOldListenersList());
return ServerImplUtils.newRaftConfiguration(conf, listener, entry.getIndex(), oldConf, oldListener);
}
+
+ /** Convert a bootstrap configuration, which has no corresponding log entry. */
+ public static RaftConfiguration toRaftConfiguration(RaftConfigurationProto proto) {
+ final List<RaftPeer> conf = ProtoUtils.toRaftPeers(proto.getPeersList());
+ final List<RaftPeer> listener = ProtoUtils.toRaftPeers(proto.getListenersList());
+ return ServerImplUtils.newRaftConfiguration(conf, listener);
+ }
}
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java
index 119f7922d..e1f60013e 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java
@@ -40,6 +40,7 @@ class RaftStorageDirectoryImpl implements RaftStorageDirectory {
private static final String IN_USE_LOCK_NAME = "in_use.lock";
private static final String META_FILE_NAME = "raft-meta";
private static final String CONF_EXTENSION = ".conf";
+ private static final String BOOTSTRAP_CONF_EXTENSION = ".bootstrap";
private static final String JVM_NAME = ManagementFactory.getRuntimeMXBean().getName();
enum StorageState {
@@ -106,6 +107,10 @@ class RaftStorageDirectoryImpl implements RaftStorageDirectory {
return new File(getCurrentDir(), META_FILE_NAME + CONF_EXTENSION);
}
+ File getBootstrapConfFile() {
+ return new File(getCurrentDir(), META_FILE_NAME + BOOTSTRAP_CONF_EXTENSION);
+ }
+
/**
* Check to see if current/ directory is empty.
*/
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
index ce809cad8..33dc333bb 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
@@ -21,6 +21,8 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.ratis.proto.RaftProtos.LogEntryProto;
+import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto;
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
import org.apache.ratis.server.RaftConfiguration;
import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy;
import org.apache.ratis.server.raftlog.LogProtoUtils;
@@ -141,16 +143,15 @@ public class RaftStorageImpl implements RaftStorage {
return metaFile.get();
}
- public void writeRaftConfiguration(LogEntryProto conf) {
+ public void writeRaftConfiguration(LogEntryProto conf) throws IOException {
File confFile = storageDir.getMetaConfFile();
try (OutputStream fio = new AtomicFileOutputStream(confFile)) {
conf.writeTo(fio);
- } catch (Exception e) {
- LOG.error("Failed writing configuration to file:" + confFile, e);
}
+ deleteBootstrapConfiguration();
}
- public RaftConfiguration readRaftConfiguration() {
+ public RaftConfiguration readRaftConfiguration() throws IOException {
File confFile = storageDir.getMetaConfFile();
if (!confFile.exists()) {
return null;
@@ -158,13 +159,50 @@ public class RaftStorageImpl implements RaftStorage {
try (InputStream fio = FileUtils.newInputStream(confFile)) {
LogEntryProto confProto = LogEntryProto.newBuilder().mergeFrom(fio).build();
return LogProtoUtils.toRaftConfiguration(confProto);
- } catch (Exception e) {
- LOG.error("Failed reading configuration from file:" + confFile, e);
- return null;
}
}
}
+ public void writeBootstrapConfiguration(RaftConfiguration conf) throws IOException {
+ if (conf.getCurrentPeers().isEmpty()
+ && conf.getCurrentPeers(RaftPeerRole.LISTENER).isEmpty()) {
+ throw new IOException("Refusing to persist an empty bootstrap configuration");
+ }
+ if (!conf.getPreviousPeers().isEmpty()
+ || !conf.getPreviousPeers(RaftPeerRole.LISTENER).isEmpty()) {
+ throw new IOException("Refusing to persist a transitional bootstrap configuration: " + conf);
+ }
+ final File confFile = storageDir.getBootstrapConfFile();
+ try (OutputStream out = new AtomicFileOutputStream(confFile)) {
+ LogProtoUtils.toRaftConfigurationProtoBuilder(conf).build().writeTo(out);
+ }
+ }
+
+ public RaftConfiguration readBootstrapConfiguration() throws IOException {
+ final File confFile = storageDir.getBootstrapConfFile();
+ if (!confFile.exists()) {
+ return null;
+ }
+ try (InputStream in = FileUtils.newInputStream(confFile)) {
+ final RaftConfigurationProto proto = RaftConfigurationProto.newBuilder().mergeFrom(in).build();
+ if (proto.getPeersCount() == 0 && proto.getListenersCount() == 0) {
+ throw new IOException("Invalid empty bootstrap configuration in " + confFile);
+ }
+ if (proto.getOldPeersCount() != 0 || proto.getOldListenersCount() != 0) {
+ throw new IOException("Invalid transitional bootstrap configuration in " + confFile);
+ }
+ try {
+ return LogProtoUtils.toRaftConfiguration(proto);
+ } catch (RuntimeException e) {
+ throw new IOException("Invalid bootstrap configuration in " + confFile, e);
+ }
+ }
+ }
+
+ public void deleteBootstrapConfiguration() throws IOException {
+ FileUtils.deleteIfExists(storageDir.getBootstrapConfFile());
+ }
+
@Override
public String toString() {
return JavaUtils.getClassSimpleName(getClass()) + ":" + getStorageDir();
diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
index a86ab2a51..12f902811 100644
--- a/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
+++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java
@@ -33,6 +33,7 @@ import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.JavaUtils;
+import org.apache.ratis.util.LifeCycle;
import org.apache.ratis.util.Slf4jUtils;
import org.apache.ratis.util.TimeDuration;
import org.apache.ratis.util.function.CheckedBiConsumer;
@@ -45,10 +46,12 @@ import org.slf4j.event.Level;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.Random;
@@ -216,6 +219,103 @@ public abstract class GroupManagementBaseTest extends BaseTest {
cluster.shutdown();
}
+ @Test
+ public void testRecoverGroupBeforeInitialConfigurationEntry() throws Exception {
+ final MiniRaftCluster cluster = getCluster(0);
+ final BlockRequestHandlingInjection injection = BlockRequestHandlingInjection.getInstance();
+
+ try {
+ // Start three servers without an initial group. The group will be added dynamically below.
+ final List<RaftPeerId> ids = Arrays.stream(MiniRaftCluster.generateIds(3, 0))
+ .map(RaftPeerId::valueOf).collect(Collectors.toList());
+ ids.forEach(id -> cluster.putNewServer(id, null, true));
+ cluster.start();
+
+ final List<RaftPeer> peers = cluster.getPeers();
+ final RaftGroup group = RaftGroup.valueOf(RaftGroupId.randomId(), peers);
+
+ // Prevent every candidate from receiving votes, so no leader can append the initial
+ // configuration entry before the servers are restarted.
+ peers.forEach(peer -> injection.blockRequestor(peer.getId().toString()));
+ try (RaftClient client = cluster.createClient(group)) {
+ for (RaftPeer peer : peers) {
+ client.getGroupManagementApi(peer.getId()).add(group);
+ }
+ }
+
+ cluster.getTimeoutMax().sleep();
+ for (RaftPeer peer : peers) {
+ final RaftServer.Division division = cluster.getDivision(peer.getId(), group.getGroupId());
+ Assertions.assertEquals(LifeCycle.State.RUNNING, division.getInfo().getLifeCycleState());
+ Assertions.assertNull(division.getRaftLog().getLastEntryTermIndex());
+ Assertions.assertFalse(division.getGroup().getPeers().isEmpty());
+ Assertions.assertTrue(new File(division.getRaftStorage().getStorageDir().getCurrentDir(),
+ "raft-meta.bootstrap").isFile());
+ }
+
+ // Restart through directory scanning: passing a null group makes recovery reconstruct
+ // the division from its directory name (group ID) only.
+ for (RaftPeer peer : peers) {
+ cluster.restartServer(peer.getId(), null, false);
+ }
+ injection.unblockAll();
+
+ cluster.getTimeoutMax().sleep();
+ for (RaftPeer peer : peers) {
+ final RaftServer.Division division = cluster.getDivision(peer.getId(), group.getGroupId());
+ Assertions.assertIterableEquals(peers, division.getGroup().getPeers());
+ Assertions.assertEquals(LifeCycle.State.RUNNING, division.getInfo().getLifeCycleState());
+ }
+ Assertions.assertNotNull(RaftTestUtil.waitForLeader(cluster, group.getGroupId()));
+ JavaUtils.attempt(() -> {
+ for (RaftPeer peer : peers) {
+ final RaftServer.Division division = cluster.getDivision(peer.getId(), group.getGroupId());
+ Assertions.assertFalse(new File(division.getRaftStorage().getStorageDir().getCurrentDir(),
+ "raft-meta.bootstrap").exists());
+ }
+ }, 10, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS),
+ "wait for bootstrap configuration cleanup", LOG);
+ } finally {
+ injection.unblockAll();
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testRejectConflictingBootstrapConfiguration() throws Exception {
+ final MiniRaftCluster cluster = getCluster(0);
+ try {
+ final List<RaftPeerId> ids = Arrays.stream(MiniRaftCluster.generateIds(3, 0))
+ .map(RaftPeerId::valueOf).collect(Collectors.toList());
+ ids.forEach(id -> cluster.putNewServer(id, null, true));
+ cluster.start();
+
+ final List<RaftPeer> peers = cluster.getPeers();
+ final RaftGroup group = RaftGroup.valueOf(RaftGroupId.randomId(), peers);
+ final RaftPeer target = peers.get(0);
+ try (RaftClient client = cluster.createClient(group)) {
+ client.getGroupManagementApi(target.getId()).add(group);
+ }
+
+ final RaftServer.Division division = cluster.getDivision(target.getId(), group.getGroupId());
+ Assertions.assertNull(division.getRaftLog().getLastEntryTermIndex());
+
+ final List<RaftPeer> conflictingPeers = new ArrayList<>(peers.subList(0, 2));
+ conflictingPeers.add(RaftPeer.newBuilder()
+ .setId("conflicting-peer")
+ .setAddress("localhost:12345")
+ .build());
+ final RaftGroup conflictingGroup = RaftGroup.valueOf(group.getGroupId(), conflictingPeers);
+ final CompletionException exception = Assertions.assertThrows(CompletionException.class,
+ () -> cluster.restartServer(target.getId(), conflictingGroup, false));
+ final Throwable cause = JavaUtils.unwrapCompletionException(exception);
+ Assertions.assertInstanceOf(IOException.class, cause);
+ Assertions.assertTrue(cause.toString().contains("Conflicting bootstrap configuration"), cause::toString);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
@Test
public void testMultiGroup5Nodes() throws Exception {
final int[] idIndex = {3, 4, 5};
diff --git a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
index 093e9add7..ca35712e7 100644
--- a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
+++ b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
@@ -24,9 +24,15 @@ import static org.apache.ratis.util.MD5FileUtil.MD5_SUFFIX;
import org.apache.ratis.BaseTest;
import org.apache.ratis.RaftTestUtil;
+import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto;
+import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
+import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.server.RaftConfiguration;
import org.apache.ratis.server.RaftServerConfigKeys;
+import org.apache.ratis.server.impl.ServerImplUtils;
import org.apache.ratis.server.protocol.TermIndex;
+import org.apache.ratis.server.raftlog.LogProtoUtils;
import org.apache.ratis.server.storage.RaftStorageDirectoryImpl.StorageState;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.SnapshotRetentionPolicy;
@@ -173,6 +179,37 @@ public class TestRaftStorage extends BaseTest {
storage.close();
}
+ @Test
+ public void testBootstrapConfiguration() throws Exception {
+ final RaftPeer follower = RaftPeer.newBuilder()
+ .setId("follower")
+ .setAddress("localhost:1001")
+ .setPriority(1)
+ .build();
+ final RaftPeer listener = RaftPeer.newBuilder()
+ .setId("listener")
+ .setAddress("localhost:1002")
+ .setStartupRole(RaftPeerRole.LISTENER)
+ .build();
+ final RaftConfiguration bootstrap = ServerImplUtils.newRaftConfiguration(
+ Collections.singletonList(follower), Collections.singletonList(listener));
+ final RaftConfigurationProto expected = LogProtoUtils.toRaftConfigurationProtoBuilder(bootstrap).build();
+
+ try (RaftStorageImpl storage = formatRaftStorage(storageDir)) {
+ storage.writeBootstrapConfiguration(bootstrap);
+ final RaftConfiguration recovered = storage.readBootstrapConfiguration();
+ Assertions.assertEquals(expected,
+ LogProtoUtils.toRaftConfigurationProtoBuilder(recovered).build());
+
+ storage.writeRaftConfiguration(LogProtoUtils.toLogEntryProto(bootstrap, 1L, 0L));
+ Assertions.assertNull(storage.readBootstrapConfiguration());
+ Assertions.assertEquals(0L, storage.readRaftConfiguration().getLogEntryIndex());
+
+ Files.write(storage.getStorageDir().getBootstrapConfFile().toPath(), new byte[0]);
+ Assertions.assertThrows(IOException.class, storage::readBootstrapConfiguration);
+ }
+ }
+
/**
* check if RaftStorage deletes tmp metafile when startup
*/