You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by sz...@apache.org on 2026/08/31 23:47:03 UTC
(ratis) branch master updated: RATIS-2680. Maintain backward compatibility for cleanupOldSnapshots (#1569)
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 86ace58a4 RATIS-2680. Maintain backward compatibility for cleanupOldSnapshots (#1569)
86ace58a4 is described below
commit 86ace58a4b6613baca81c775a62bc306c7cd6e6c
Author: Rui Wang <am...@apache.org>
AuthorDate: Tue Sep 1 07:46:58 2026 +0800
RATIS-2680. Maintain backward compatibility for cleanupOldSnapshots (#1569)
---
.../impl/SimpleStateMachineStorage.java | 10 ++
.../ratis/server/storage/TestRaftStorage.java | 122 ++++++++++++++++++++-
2 files changed, 131 insertions(+), 1 deletion(-)
diff --git a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
index 532277899..79d6d4582 100644
--- a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
+++ b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
@@ -120,11 +120,13 @@ public class SimpleStateMachineStorage implements StateMachineStorage {
final List<SingleFileSnapshotInfo> allSnapshotFiles = getSingleFileSnapshotInfos(stateMachineDir.toPath());
allSnapshotFiles.sort(Comparator.comparing(SingleFileSnapshotInfo::getIndex).reversed());
int numSnapshotsWithMd5 = 0;
+ int lastSnapshotsWithMd5 = -1;
int deleteIdx = -1;
for (int i = 0; i < allSnapshotFiles.size(); i++) {
final SingleFileSnapshotInfo snapshot = allSnapshotFiles.get(i);
if (snapshot.hasMd5()) {
+ lastSnapshotsWithMd5 = i;
if (++numSnapshotsWithMd5 == numSnapshotsRetained) {
// We have found the last snapshot with an MD5 file that needs to be retained
deleteIdx = i + 1;
@@ -135,6 +137,14 @@ public class SimpleStateMachineStorage implements StateMachineStorage {
}
}
+ if (deleteIdx == -1 && allSnapshotFiles.size() > numSnapshotsRetained) {
+ // Before RATIS-244, the newest numSnapshotsRetained snapshots (with or without md5) are retained.
+ // For backward compatibility:
+ // - All snapshots without MD5 : retain the newest numSnapshotsRetained snapshots.
+ // - Snapshots mixed (with and without md5): retain all snapshots with md5 and the newer snapshots without md5.
+ deleteIdx = Math.max(numSnapshotsRetained, lastSnapshotsWithMd5 + 1);
+ }
+
if (deleteIdx > 0) {
allSnapshotFiles.subList(deleteIdx, allSnapshotFiles.size())
.stream()
diff --git a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
index ca35712e7..5c31bb7aa 100644
--- a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
+++ b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
@@ -491,13 +491,133 @@ public class TestRaftStorage extends BaseTest {
simpleStateMachineStorage.cleanupOldSnapshots(snapshotRetentionPolicy);
List<String> snapshotNames = listMatchingFileNames(stateMachineDir, SNAPSHOT_REGEX);
- Assertions.assertEquals(3, snapshotNames.size());
+ Assertions.assertEquals(2, snapshotNames.size());
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 300)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 200)));
+ Assertions.assertFalse(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 100)));
Assertions.assertTrue(listMatchingFileNames(stateMachineDir, SNAPSHOT_MD5_REGEX).isEmpty());
} finally {
storage.close();
}
}
+ @Test
+ public void testCleanupOldSnapshotsMixedRetainsAllMd5AndNewerWithoutMd5() throws Exception {
+ SnapshotRetentionPolicy snapshotRetentionPolicy = new SnapshotRetentionPolicy() {
+ @Override
+ public int getNumSnapshotsRetained() {
+ return 3;
+ }
+ };
+
+ SimpleStateMachineStorage simpleStateMachineStorage = new SimpleStateMachineStorage();
+ final RaftStorage storage = newRaftStorage(storageDir);
+ simpleStateMachineStorage.init(storage);
+ try {
+ // Newer snapshots without MD5 (pre-upgrade) at higher indices.
+ createSnapshot(simpleStateMachineStorage, 1, 900, false);
+ createSnapshot(simpleStateMachineStorage, 1, 800, false);
+ // Newer snapshots with MD5 (post-upgrade) at lower indices; fewer than numSnapshotsRetained.
+ createSnapshot(simpleStateMachineStorage, 1, 700, true);
+ createSnapshot(simpleStateMachineStorage, 1, 600, true);
+
+ File stateMachineDir = storage.getStorageDir().getStateMachineDir();
+ simpleStateMachineStorage.cleanupOldSnapshots(snapshotRetentionPolicy);
+
+ // Fallback retains all MD5 snapshots and the newer snapshots without MD5.
+ List<String> snapshotNames = listMatchingFileNames(stateMachineDir, SNAPSHOT_REGEX);
+ Assertions.assertEquals(4, snapshotNames.size());
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 900)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 800)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 700)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 600)));
+
+ List<String> md5Names = listMatchingFileNames(stateMachineDir, SNAPSHOT_MD5_REGEX);
+ Assertions.assertEquals(2, md5Names.size());
+ Assertions.assertTrue(md5Names.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 700) + MD5_SUFFIX));
+ Assertions.assertTrue(md5Names.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 600) + MD5_SUFFIX));
+ } finally {
+ storage.close();
+ }
+ }
+
+ @Test
+ public void testCleanupOldSnapshotsMixedUpgradeScenario() throws Exception {
+ SnapshotRetentionPolicy snapshotRetentionPolicy = new SnapshotRetentionPolicy() {
+ @Override
+ public int getNumSnapshotsRetained() {
+ return 3;
+ }
+ };
+
+ SimpleStateMachineStorage simpleStateMachineStorage = new SimpleStateMachineStorage();
+ final RaftStorage storage = newRaftStorage(storageDir);
+ simpleStateMachineStorage.init(storage);
+ try {
+ // Old snapshots without MD5 created before upgrade.
+ createSnapshot(simpleStateMachineStorage, 1, 500, false);
+ createSnapshot(simpleStateMachineStorage, 1, 600, false);
+ createSnapshot(simpleStateMachineStorage, 1, 700, false);
+ // New snapshots with MD5 created after upgrade; fewer than numSnapshotsRetained.
+ createSnapshot(simpleStateMachineStorage, 1, 800, true);
+ createSnapshot(simpleStateMachineStorage, 1, 900, true);
+
+ File stateMachineDir = storage.getStorageDir().getStateMachineDir();
+ simpleStateMachineStorage.cleanupOldSnapshots(snapshotRetentionPolicy);
+
+ // Retain all MD5 snapshots plus the newest old snapshot without MD5.
+ List<String> snapshotNames = listMatchingFileNames(stateMachineDir, SNAPSHOT_REGEX);
+ Assertions.assertEquals(3, snapshotNames.size());
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 900)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 800)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 700)));
+ Assertions.assertFalse(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 600)));
+ Assertions.assertFalse(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 500)));
+
+ List<String> md5Names = listMatchingFileNames(stateMachineDir, SNAPSHOT_MD5_REGEX);
+ Assertions.assertEquals(2, md5Names.size());
+ } finally {
+ storage.close();
+ }
+ }
+
+ @Test
+ public void testCleanupOldSnapshotsMixedDeletesOlderWithoutMd5BeyondRetention() throws Exception {
+ SnapshotRetentionPolicy snapshotRetentionPolicy = new SnapshotRetentionPolicy() {
+ @Override
+ public int getNumSnapshotsRetained() {
+ return 3;
+ }
+ };
+
+ SimpleStateMachineStorage simpleStateMachineStorage = new SimpleStateMachineStorage();
+ final RaftStorage storage = newRaftStorage(storageDir);
+ simpleStateMachineStorage.init(storage);
+ try {
+ createSnapshot(simpleStateMachineStorage, 1, 900, false);
+ createSnapshot(simpleStateMachineStorage, 1, 700, true);
+ createSnapshot(simpleStateMachineStorage, 1, 600, true);
+ createSnapshot(simpleStateMachineStorage, 1, 500, false);
+ createSnapshot(simpleStateMachineStorage, 1, 400, false);
+
+ File stateMachineDir = storage.getStorageDir().getStateMachineDir();
+ simpleStateMachineStorage.cleanupOldSnapshots(snapshotRetentionPolicy);
+
+ List<String> snapshotNames = listMatchingFileNames(stateMachineDir, SNAPSHOT_REGEX);
+ Assertions.assertEquals(3, snapshotNames.size());
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 900)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 700)));
+ Assertions.assertTrue(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 600)));
+ Assertions.assertFalse(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 500)));
+ Assertions.assertFalse(snapshotNames.contains(SimpleStateMachineStorage.getSnapshotFileName(1, 400)));
+
+ List<String> md5Names = listMatchingFileNames(stateMachineDir, SNAPSHOT_MD5_REGEX);
+ Assertions.assertEquals(2, md5Names.size());
+ } finally {
+ storage.close();
+ }
+ }
+
@Test
public void testGetLatestSnapshotReturnsNewest() throws Exception {
SimpleStateMachineStorage simpleStateMachineStorage = new SimpleStateMachineStorage();