You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2025/10/15 11:40:08 UTC
(iotdb) 01/01: Add config "keep_same_disk_when_loading_snapshot"
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch add_keep_same_disk_when_loading_snapshot
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b5ec2cafebe73129f8d776bed400d98b718f976e
Author: Tian Jiang <jt...@163.com>
AuthorDate: Wed Oct 15 19:39:53 2025 +0800
Add config "keep_same_disk_when_loading_snapshot"
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++
.../dataregion/snapshot/SnapshotLoader.java | 99 ++++++++++--------
.../storageengine/rescon/disk/FolderManager.java | 22 ++++
.../dataregion/snapshot/IoTDBSnapshotTest.java | 111 ++++++++++++++++++---
.../conf/iotdb-system.properties.template | 6 ++
6 files changed, 199 insertions(+), 55 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 74a966eacea..11e3610e1ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1176,6 +1176,8 @@ public class IoTDBConfig {
private boolean includeNullValueInWriteThroughputMetric = false;
+ private boolean keepSameDiskWhenLoadingSnapshot = false;
+
IoTDBConfig() {}
public int getMaxLogEntriesNumPerBatch() {
@@ -4225,4 +4227,12 @@ public class IoTDBConfig {
public void setPasswordLockTimeMinutes(int passwordLockTimeMinutes) {
this.passwordLockTimeMinutes = passwordLockTimeMinutes;
}
+
+ public boolean isKeepSameDiskWhenLoadingSnapshot() {
+ return keepSameDiskWhenLoadingSnapshot;
+ }
+
+ public void setKeepSameDiskWhenLoadingSnapshot(boolean keepSameDiskWhenLoadingSnapshot) {
+ this.keepSameDiskWhenLoadingSnapshot = keepSameDiskWhenLoadingSnapshot;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 0fa4aaf90e7..cbb92d22959 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1171,6 +1171,12 @@ public class IoTDBDescriptor {
"region_migration_speed_limit_bytes_per_second",
ConfigurationFileUtils.getConfigurationDefaultValue(
"region_migration_speed_limit_bytes_per_second"))));
+ conf.setKeepSameDiskWhenLoadingSnapshot(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "keep_same_disk_when_loading_snapshot",
+ ConfigurationFileUtils.getConfigurationDefaultValue(
+ "keep_same_disk_when_loading_snapshot"))));
}
private void loadIoTConsensusV2Props(TrimProperties properties) throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
index daa64274b12..ffcee4f51bd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
@@ -314,56 +314,69 @@ public class SnapshotLoader {
}
}
+ private File createLinksFromSnapshotToSourceDir(
+ String targetSuffix,
+ File file,
+ Map<String, String> fileTarget,
+ String fileKey,
+ String finalDir)
+ throws IOException {
+ File targetFile =
+ new File(finalDir + File.separator + targetSuffix + File.separator + file.getName());
+
+ try {
+ if (!targetFile.getParentFile().exists() && !targetFile.getParentFile().mkdirs()) {
+ throw new IOException(
+ String.format(
+ "Cannot create directory %s", targetFile.getParentFile().getAbsolutePath()));
+ }
+
+ try {
+ Files.createLink(targetFile.toPath(), file.toPath());
+ LOGGER.debug("Created hard link from {} to {}", file, targetFile);
+ fileTarget.put(fileKey, finalDir);
+ return targetFile;
+ } catch (IOException e) {
+ LOGGER.info("Cannot create link from {} to {}, fallback to copy", file, targetFile);
+ }
+
+ Files.copy(file.toPath(), targetFile.toPath());
+ fileTarget.put(fileKey, finalDir);
+ return targetFile;
+ } catch (Exception e) {
+ LOGGER.warn(
+ "Failed to process file {} in dir {}: {}", file.getName(), finalDir, e.getMessage(), e);
+ throw e;
+ }
+ }
+
private void createLinksFromSnapshotToSourceDir(
- String targetSuffix, File[] files, FolderManager folderManager)
- throws DiskSpaceInsufficientException, IOException {
+ String targetSuffix, File[] files, FolderManager folderManager) throws IOException {
Map<String, String> fileTarget = new HashMap<>();
for (File file : files) {
String fileKey = file.getName().split("\\.")[0];
String dataDir = fileTarget.get(fileKey);
+ if (dataDir != null) {
+ createLinksFromSnapshotToSourceDir(targetSuffix, file, fileTarget, fileKey, dataDir);
+ continue;
+ }
+
try {
- folderManager.getNextWithRetry(
- currentDataDir -> {
- String effectiveDir = (dataDir != null) ? dataDir : currentDataDir;
- File targetFile =
- new File(
- effectiveDir
- + File.separator
- + targetSuffix
- + File.separator
- + file.getName());
-
- try {
- if (!targetFile.getParentFile().exists() && !targetFile.getParentFile().mkdirs()) {
- throw new IOException(
- String.format(
- "Cannot create directory %s",
- targetFile.getParentFile().getAbsolutePath()));
- }
-
- try {
- Files.createLink(targetFile.toPath(), file.toPath());
- LOGGER.debug("Created hard link from {} to {}", file, targetFile);
- return targetFile;
- } catch (IOException e) {
- LOGGER.info(
- "Cannot create link from {} to {}, fallback to copy", file, targetFile);
- }
-
- Files.copy(file.toPath(), targetFile.toPath());
- fileTarget.put(fileKey, effectiveDir);
- return targetFile;
- } catch (Exception e) {
- LOGGER.warn(
- "Failed to process file {} in dir {}: {}",
- file.getName(),
- effectiveDir,
- e.getMessage(),
- e);
- throw e;
- }
- });
+ String firstFolderOfSameDisk =
+ IoTDBDescriptor.getInstance().getConfig().isKeepSameDiskWhenLoadingSnapshot()
+ ? folderManager.getFirstFolderOfSameDisk(file.getAbsolutePath())
+ : null;
+
+ if (firstFolderOfSameDisk != null) {
+ createLinksFromSnapshotToSourceDir(
+ targetSuffix, file, fileTarget, fileKey, firstFolderOfSameDisk);
+ } else {
+ folderManager.getNextWithRetry(
+ currentDataDir ->
+ createLinksFromSnapshotToSourceDir(
+ targetSuffix, file, fileTarget, fileKey, currentDataDir));
+ }
} catch (Exception e) {
throw new IOException(
String.format(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
index e90292853f1..8d3b15d72b5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
@@ -32,6 +32,11 @@ import org.apache.iotdb.db.storageengine.rescon.disk.strategy.SequenceStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -146,4 +151,21 @@ public class FolderManager {
public List<String> getFolders() {
return folders;
}
+
+ public String getFirstFolderOfSameDisk(String pathStr) {
+ Path path = Paths.get(pathStr);
+ try {
+ FileStore fileStore = Files.getFileStore(path);
+ for (String folder : folders) {
+ Path folderPath = Paths.get(folder);
+ FileStore folderFileStore = Files.getFileStore(folderPath);
+ if (folderFileStore.equals(fileStore)) {
+ return folder;
+ }
+ }
+ } catch (IOException e) {
+ logger.warn("Failed to read file store path '" + pathStr + "'", e);
+ }
+ return null;
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java
index 6bec0df050b..926352646aa 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java
@@ -38,15 +38,21 @@ import org.apache.tsfile.utils.TsFileGeneratorUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.iotdb.consensus.iot.IoTConsensusServerImpl.SNAPSHOT_DIR_NAME;
import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+import static org.junit.Assert.assertEquals;
public class IoTDBSnapshotTest {
private String[][] testDataDirs =
@@ -65,11 +71,12 @@ public class IoTDBSnapshotTest {
FileUtils.recursivelyDeleteFolder("target" + File.separator + "tmp");
}
- private List<TsFileResource> writeTsFiles() throws IOException, WriteProcessException {
+ private List<TsFileResource> writeTsFiles(String[] dataDirs)
+ throws IOException, WriteProcessException {
List<TsFileResource> resources = new ArrayList<>();
for (int i = 0; i < 100; i++) {
String filePath =
- testDataDirs[0][i % 3]
+ dataDirs[i % dataDirs.length]
+ File.separator
+ "sequence"
+ File.separator
@@ -108,7 +115,7 @@ public class IoTDBSnapshotTest {
IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs);
TierManager.getInstance().resetFolders();
try {
- List<TsFileResource> resources = writeTsFiles();
+ List<TsFileResource> resources = writeTsFiles(testDataDirs[0]);
DataRegion region = new DataRegion(testSgName, "0");
region.getTsFileManager().addAll(resources, true);
File snapshotDir = new File("target" + File.separator + "snapshot");
@@ -117,12 +124,12 @@ public class IoTDBSnapshotTest {
new SnapshotTaker(region).takeFullSnapshot(snapshotDir.getAbsolutePath(), true);
File[] files =
snapshotDir.listFiles((dir, name) -> name.equals(SnapshotLogger.SNAPSHOT_LOG_NAME));
- Assert.assertEquals(1, files.length);
+ assertEquals(1, files.length);
SnapshotLogAnalyzer analyzer = new SnapshotLogAnalyzer(files[0]);
Assert.assertTrue(analyzer.isSnapshotComplete());
int cnt = analyzer.getTotalFileCountInSnapshot();
analyzer.close();
- Assert.assertEquals(200, cnt);
+ assertEquals(200, cnt);
for (TsFileResource resource : resources) {
Assert.assertTrue(resource.tryWriteLock());
}
@@ -142,7 +149,7 @@ public class IoTDBSnapshotTest {
IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs);
TierManager.getInstance().resetFolders();
try {
- List<TsFileResource> resources = writeTsFiles();
+ List<TsFileResource> resources = writeTsFiles(testDataDirs[0]);
resources.subList(50, 100).forEach(x -> x.setStatusForTest(TsFileResourceStatus.UNCLOSED));
DataRegion region = new DataRegion(testSgName, "0");
region.setAllowCompaction(false);
@@ -153,13 +160,13 @@ public class IoTDBSnapshotTest {
new SnapshotTaker(region).takeFullSnapshot(snapshotDir.getAbsolutePath(), true);
File[] files =
snapshotDir.listFiles((dir, name) -> name.equals(SnapshotLogger.SNAPSHOT_LOG_NAME));
- Assert.assertEquals(1, files.length);
+ assertEquals(1, files.length);
SnapshotLogAnalyzer analyzer = new SnapshotLogAnalyzer(files[0]);
int cnt = 0;
Assert.assertTrue(analyzer.isSnapshotComplete());
cnt = analyzer.getTotalFileCountInSnapshot();
analyzer.close();
- Assert.assertEquals(100, cnt);
+ assertEquals(100, cnt);
for (TsFileResource resource : resources) {
Assert.assertTrue(resource.tryWriteLock());
}
@@ -179,7 +186,7 @@ public class IoTDBSnapshotTest {
IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs);
TierManager.getInstance().resetFolders();
try {
- List<TsFileResource> resources = writeTsFiles();
+ List<TsFileResource> resources = writeTsFiles(testDataDirs[0]);
DataRegion region = new DataRegion(testSgName, "0");
CompressionRatio.getInstance().updateRatio(100, 100, "0");
region.getTsFileManager().addAll(resources, true);
@@ -195,8 +202,8 @@ public class IoTDBSnapshotTest {
.loadSnapshotForStateMachine();
Assert.assertNotNull(dataRegion);
List<TsFileResource> resource = dataRegion.getTsFileManager().getTsFileList(true);
- Assert.assertEquals(100, resource.size());
- Assert.assertEquals(
+ assertEquals(100, resource.size());
+ assertEquals(
new Pair<>(100L, 100L),
CompressionRatio.getInstance().getDataRegionRatioMap().get("0"));
} finally {
@@ -208,6 +215,86 @@ public class IoTDBSnapshotTest {
}
}
+ @Ignore("Need manual execution to specify different disks")
+ @Test
+ public void testLoadSnapshotNoHardLink()
+ throws IOException, WriteProcessException, DirectoryNotLegalException {
+ IoTDBDescriptor.getInstance().getConfig().setKeepSameDiskWhenLoadingSnapshot(true);
+ // initialize dirs
+ String[][] dataDirsForDB = new String[][] {{"C://snapshot_test", "D://snapshot_test"}};
+ File snapshotDir = new File("D://snapshot_store//");
+ if (snapshotDir.exists()) {
+ FileUtils.recursivelyDeleteFolder(snapshotDir.getAbsolutePath());
+ }
+ for (String[] dirs : dataDirsForDB) {
+ for (String dir : dirs) {
+ if (new File(dir).exists()) {
+ FileUtils.recursivelyDeleteFolder(dir);
+ }
+ }
+ }
+ IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(dataDirsForDB);
+ TierManager.getInstance().resetFolders();
+
+ // prepare files, files should be written into two folders
+ List<TsFileResource> resources = writeTsFiles(dataDirsForDB[0]);
+ DataRegion region = new DataRegion(testSgName, "0");
+ region.getTsFileManager().addAll(resources, true);
+
+ // take a snapshot into one disk
+ Assert.assertTrue(snapshotDir.exists() || snapshotDir.mkdirs());
+ try {
+ Assert.assertTrue(
+ new SnapshotTaker(region).takeFullSnapshot(snapshotDir.getAbsolutePath(), true));
+ File[] files =
+ snapshotDir.listFiles((dir, name) -> name.equals(SnapshotLogger.SNAPSHOT_LOG_NAME));
+ // use loadWithoutLog
+ if (files != null && files.length > 0) {
+ files[0].delete();
+ }
+ // move files to snapshot store (simulate snapshot transfer)
+ for (String dir : dataDirsForDB[0]) {
+ File internalSnapshotDir = new File(dir, SNAPSHOT_DIR_NAME);
+ if (internalSnapshotDir.exists()) {
+ for (File file : FileUtils.listFilesRecursively(internalSnapshotDir, f -> true)) {
+ if (file.isFile()) {
+ String absolutePath = file.getAbsolutePath();
+ int snapshotIdIndex = absolutePath.indexOf("snapshot_store");
+ int suffixIndex = snapshotIdIndex + "snapshot_store".length();
+ String suffix = absolutePath.substring(suffixIndex);
+ File snapshotFile = new File(snapshotDir, suffix);
+ FileUtils.copyFile(file, snapshotFile);
+ }
+ }
+ }
+ }
+
+ // load the snapshot
+ DataRegion dataRegion =
+ new SnapshotLoader(snapshotDir.getAbsolutePath(), testSgName, "0")
+ .loadSnapshotForStateMachine();
+ Assert.assertNotNull(dataRegion);
+ resources = dataRegion.getTsFileManager().getTsFileList(true);
+ assertEquals(100, resources.size());
+
+ // files should not be moved to another disk
+ Path snapshotDirPath = snapshotDir.toPath();
+ FileStore snapshotFileStore = Files.getFileStore(snapshotDirPath);
+ for (TsFileResource tsFileResource : resources) {
+ Path tsfilePath = tsFileResource.getTsFile().toPath();
+ FileStore tsFileFileStore = Files.getFileStore(tsfilePath);
+ assertEquals(snapshotFileStore, tsFileFileStore);
+ }
+ } finally {
+ FileUtils.recursivelyDeleteFolder(snapshotDir.getAbsolutePath());
+ for (String[] dirs : dataDirsForDB) {
+ for (String dir : dirs) {
+ FileUtils.recursivelyDeleteFolder(dir);
+ }
+ }
+ }
+ }
+
@Test
public void testGetSnapshotFile() throws IOException {
File tsFile =
@@ -228,7 +315,7 @@ public class IoTDBSnapshotTest {
Mockito.when(region.getDataRegionId()).thenReturn("0");
File snapshotFile =
new SnapshotTaker(region).getSnapshotFilePathForTsFile(tsFile, "test-snapshotId");
- Assert.assertEquals(
+ assertEquals(
new File(
IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs()[0]
+ File.separator
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 207c0507093..eb55c021d7a 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1607,6 +1607,12 @@ data_region_iot_max_memory_ratio_for_queue = 0.6
# Datatype: long
region_migration_speed_limit_bytes_per_second = 50331648
+# When loading snapshot, try keeping TsFiles in the same disk as the snapshot dir.
+# This may reduce file copies but may also result in a worse disk load-balance
+# effectiveMode: hot_reload
+# Datatype: boolean
+keep_same_disk_when_loading_snapshot=false
+
####################
### Blob Allocator Configuration
####################