You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by sz...@apache.org on 2021/12/29 10:35:24 UTC
[ratis] branch master updated: RATIS-1479. Make the min gap to take snapshot configurable (#572)
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 94db58b RATIS-1479. Make the min gap to take snapshot configurable (#572)
94db58b is described below
commit 94db58b9fc5a855beba51fdd04e31f899ef96abc
Author: Yaolong Liu <ly...@163.com>
AuthorDate: Wed Dec 29 18:35:18 2021 +0800
RATIS-1479. Make the min gap to take snapshot configurable (#572)
---
.../apache/ratis/server/RaftServerConfigKeys.java | 12 +++++++
.../apache/ratis/server/impl/RaftServerImpl.java | 10 +++---
.../ratis/statemachine/SnapshotManagementTest.java | 38 ++++++++++++++++++++--
3 files changed, 53 insertions(+), 7 deletions(-)
diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java
index 51178d3..309bf1d 100644
--- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java
+++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java
@@ -433,6 +433,18 @@ public interface RaftServerConfigKeys {
setBoolean(properties::setBoolean, AUTO_TRIGGER_ENABLED_KEY, autoTriggerThreshold);
}
+ /** The log index gap between to two snapshot creations. */
+ String CREATION_GAP_KEY = PREFIX + ".creation.gap";
+ long CREATION_GAP_DEFAULT = 1024;
+ static long creationGap(RaftProperties properties) {
+ return getLong(
+ properties::getLong, CREATION_GAP_KEY, CREATION_GAP_DEFAULT,
+ getDefaultLog(), requireMin(1L));
+ }
+ static void setCreationGap(RaftProperties properties, long creationGap) {
+ setLong(properties::setLong, CREATION_GAP_KEY, creationGap);
+ }
+
/** log size limit (in number of log entries) that triggers the snapshot */
String AUTO_TRIGGER_THRESHOLD_KEY = PREFIX + ".auto.trigger.threshold";
long AUTO_TRIGGER_THRESHOLD_DEFAULT = 400000L;
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
index 6e2ac4d..fdba1d7 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
@@ -979,12 +979,12 @@ class RaftServerImpl implements RaftServer.Division,
assertLifeCycleState(LifeCycle.States.RUNNING);
assertGroup(request.getRequestorId(), request.getRaftGroupId());
- //TODO(liuyaolong): make the min gap configurable, or get the gap value from shell command
- long minGapValue = 5;
- final Long lastSnapshotIndex = Optional.ofNullable(stateMachine.getLatestSnapshot())
+ //TODO(liuyaolong): get the gap value from shell command
+ long minGapValue = RaftServerConfigKeys.Snapshot.creationGap(proxy.getProperties());
+ final long lastSnapshotIndex = Optional.ofNullable(stateMachine.getLatestSnapshot())
.map(SnapshotInfo::getIndex)
- .orElse(null);
- if (lastSnapshotIndex != null && state.getLastAppliedIndex() - lastSnapshotIndex < minGapValue) {
+ .orElse(0L);
+ if (state.getLastAppliedIndex() - lastSnapshotIndex < minGapValue) {
return CompletableFuture.completedFuture(newSuccessReply(request, lastSnapshotIndex));
}
diff --git a/ratis-server/src/test/java/org/apache/ratis/statemachine/SnapshotManagementTest.java b/ratis-server/src/test/java/org/apache/ratis/statemachine/SnapshotManagementTest.java
index 47a19ba..f941458 100644
--- a/ratis-server/src/test/java/org/apache/ratis/statemachine/SnapshotManagementTest.java
+++ b/ratis-server/src/test/java/org/apache/ratis/statemachine/SnapshotManagementTest.java
@@ -56,12 +56,14 @@ public abstract class SnapshotManagementTest<CLUSTER extends MiniRaftCluster>
final RaftProperties p = getProperties();
p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
SimpleStateMachine4Testing.class, StateMachine.class);
+ RaftServerConfigKeys.Snapshot.setCreationGap(p,20L);
RaftServerConfigKeys.Snapshot.setAutoTriggerEnabled(p, false);
}
@Test
public void testTakeSnapshot() throws Exception {
runWithNewCluster(1, this::runTestTakeSnapshot);
+ runWithNewCluster(1,this::runTestTakeSnapshotWithConfigurableGap);
}
void runTestTakeSnapshot(CLUSTER cluster) throws Exception {
@@ -69,8 +71,7 @@ public abstract class SnapshotManagementTest<CLUSTER extends MiniRaftCluster>
final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster);
final RaftPeerId leaderId = leader.getId();
try (final RaftClient client = cluster.createClient(leaderId)) {
- //todo(yaolong liu) : make 5 to be a configurable value
- for (int i = 0; i < 5; i++) {
+ for (int i = 0; i < RaftServerConfigKeys.Snapshot.creationGap(getProperties()); i++) {
RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i));
Assert.assertTrue(reply.isSuccess());
}
@@ -87,4 +88,37 @@ public abstract class SnapshotManagementTest<CLUSTER extends MiniRaftCluster>
.getStateMachineStorage().getSnapshotFile(leader.getInfo().getCurrentTerm(), snapshotIndex);
Assert.assertTrue(snapshotFile.exists());
}
+
+ void runTestTakeSnapshotWithConfigurableGap(CLUSTER cluster) throws Exception {
+ RaftClientReply snapshotReply;
+ final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster);
+ final RaftPeerId leaderId = leader.getId();
+ try (final RaftClient client = cluster.createClient(leaderId)) {
+ for (int i = 0; i < RaftServerConfigKeys.Snapshot.creationGap(getProperties())/2-1; i++) {
+ RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i));
+ Assert.assertTrue(reply.isSuccess());
+ }
+ Assert.assertTrue(leader.getStateMachine().getLastAppliedTermIndex().getIndex()
+ < RaftServerConfigKeys.Snapshot.creationGap(getProperties()));
+ final SnapshotRequest r = new SnapshotRequest(client.getId(), leaderId, cluster.getGroupId(),
+ CallId.getAndIncrement(), 3000);
+ snapshotReply = RaftServerTestUtil.takeSnapshotAsync(leader, r).join();
+ Assert.assertTrue(snapshotReply.isSuccess());
+ Assert.assertEquals(0,snapshotReply.getLogIndex());
+ for (int i = 0; i < RaftServerConfigKeys.Snapshot.creationGap(getProperties())/2-1; i++) {
+ RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i));
+ Assert.assertTrue(reply.isSuccess());
+ }
+ final SnapshotRequest r1 = new SnapshotRequest(client.getId(), leaderId, cluster.getGroupId(),
+ CallId.getAndIncrement(), 3000);
+ snapshotReply = RaftServerTestUtil.takeSnapshotAsync(leader, r1).join();
+ }
+ Assert.assertTrue(snapshotReply.isSuccess());
+ final long snapshotIndex = snapshotReply.getLogIndex();
+ LOG.info("snapshotIndex = {}", snapshotIndex);
+
+ final File snapshotFile = SimpleStateMachine4Testing.get(leader)
+ .getStateMachineStorage().getSnapshotFile(leader.getInfo().getCurrentTerm(), snapshotIndex);
+ Assert.assertTrue(snapshotFile.exists());
+ }
}