You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2026/07/01 08:41:22 UTC
(iotdb) branch cp-pr-18045-dev-1.3 created (now 8ac47dc49b8)
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a change to branch cp-pr-18045-dev-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 8ac47dc49b8 Replace remaining compaction hard links with FileUtils
This branch includes the following new commits:
new e09546ec350 Add copy fallback for compaction hard links
new 0c6aa7c65eb Simplify create link fallback handling
new 8ac47dc49b8 Replace remaining compaction hard links with FileUtils
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 02/03: Simplify create link fallback handling
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch cp-pr-18045-dev-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0c6aa7c65eb565feae888a11bd0fbe0426bfc5c9
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Fri Jun 26 18:30:01 2026 +0800
Simplify create link fallback handling
---
.../main/java/org/apache/iotdb/commons/utils/FileUtils.java | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
index ee5bff42ad6..01232f9237a 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
@@ -74,17 +74,7 @@ public class FileUtils {
throws IOException {
try {
Files.createLink(link, existing);
- } catch (IOException e) {
- if (!fallBackToCopy) {
- throw e;
- }
- try {
- Files.copy(existing, link);
- } catch (IOException copyException) {
- copyException.addSuppressed(e);
- throw copyException;
- }
- } catch (UnsupportedOperationException e) {
+ } catch (IOException | UnsupportedOperationException e) {
if (!fallBackToCopy) {
throw e;
}
(iotdb) 01/03: Add copy fallback for compaction hard links
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch cp-pr-18045-dev-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e09546ec3507ca76f668664e5ac5d0c28490e398
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Fri Jun 26 18:25:23 2026 +0800
Add copy fallback for compaction hard links
---
.../RepairUnsortedFileCompactionPerformer.java | 9 ++++----
.../task/InsertionCrossSpaceCompactionTask.java | 14 ++++++-----
.../dataregion/modification/ModificationFile.java | 3 ++-
.../org/apache/iotdb/commons/utils/FileUtils.java | 27 ++++++++++++++++++++++
4 files changed, 42 insertions(+), 11 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
index 18e9ef24df2..534ea471ffa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.RepairUnsortedFileCompactionWriter;
@@ -31,7 +32,6 @@ import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.Collections;
import java.util.List;
@@ -74,7 +74,7 @@ public class RepairUnsortedFileCompactionPerformer extends ReadPointCompactionPe
private void prepareTargetFile() throws IOException {
TsFileResource seqSourceFile = seqFiles.get(0);
TsFileResource targetFile = targetFiles.get(0);
- Files.createLink(targetFile.getTsFile().toPath(), seqSourceFile.getTsFile().toPath());
+ FileUtils.createLink(targetFile.getTsFile().toPath(), seqSourceFile.getTsFile().toPath(), true);
ITimeIndex timeIndex = seqSourceFile.getTimeIndex();
if (timeIndex instanceof DeviceTimeIndex) {
targetFile.setTimeIndex(timeIndex);
@@ -82,9 +82,10 @@ public class RepairUnsortedFileCompactionPerformer extends ReadPointCompactionPe
targetFile.setTimeIndex(CompactionUtils.buildDeviceTimeIndex(seqSourceFile));
}
if (seqSourceFile.modFileExists()) {
- Files.createLink(
+ FileUtils.createLink(
new File(seqSourceFile.getCompactionModFile().getFilePath()).toPath(),
- new File(seqSourceFile.getModFile().getFilePath()).toPath());
+ new File(seqSourceFile.getModFile().getFilePath()).toPath(),
+ true);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index 992b5406708..e4440ac32f9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
@@ -37,7 +38,6 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameG
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -218,14 +218,16 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
private void prepareTargetFiles() throws IOException {
File sourceTsFile = unseqFileToInsert.getTsFile();
File targetTsFile = targetFile.getTsFile();
- Files.createLink(targetTsFile.toPath(), sourceTsFile.toPath());
- Files.createLink(
+ FileUtils.createLink(targetTsFile.toPath(), sourceTsFile.toPath(), true);
+ FileUtils.createLink(
new File(targetTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
+ true);
if (unseqFileToInsert.getModFile().exists()) {
- Files.createLink(
+ FileUtils.createLink(
new File(targetTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
+ true);
}
targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndex());
targetFile.deserialize();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
index 6489c7e5299..bab214c5974 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.modification;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.storageengine.dataregion.modification.io.LocalTextModificationAccessor;
import org.apache.iotdb.db.storageengine.dataregion.modification.io.ModificationReader;
import org.apache.iotdb.db.storageengine.dataregion.modification.io.ModificationWriter;
@@ -180,7 +181,7 @@ public class ModificationFile implements AutoCloseable {
File hardlink = new File(filePath + hardlinkSuffix);
try {
- Files.createLink(Paths.get(hardlink.getAbsolutePath()), Paths.get(filePath));
+ FileUtils.createLink(Paths.get(hardlink.getAbsolutePath()), Paths.get(filePath), true);
return new ModificationFile(hardlink.getAbsolutePath());
} catch (FileAlreadyExistsException e) {
// retry a different name if the file is already created
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
index 557adf43f62..ee5bff42ad6 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
@@ -70,6 +70,33 @@ public class FileUtils {
}
}
+ public static void createLink(Path link, Path existing, boolean fallBackToCopy)
+ throws IOException {
+ try {
+ Files.createLink(link, existing);
+ } catch (IOException e) {
+ if (!fallBackToCopy) {
+ throw e;
+ }
+ try {
+ Files.copy(existing, link);
+ } catch (IOException copyException) {
+ copyException.addSuppressed(e);
+ throw copyException;
+ }
+ } catch (UnsupportedOperationException e) {
+ if (!fallBackToCopy) {
+ throw e;
+ }
+ try {
+ Files.copy(existing, link);
+ } catch (IOException copyException) {
+ copyException.addSuppressed(e);
+ throw copyException;
+ }
+ }
+ }
+
public static void deleteFileOrDirectory(File file) {
deleteFileOrDirectory(file, false);
}
(iotdb) 03/03: Replace remaining compaction hard links with FileUtils
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch cp-pr-18045-dev-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8ac47dc49b867466168350997ce7a1b93fe5161d
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Wed Jul 1 16:40:58 2026 +0800
Replace remaining compaction hard links with FileUtils
---
.../execute/task/InnerSpaceCompactionTask.java | 13 +++++----
.../task/RepairUnsortedFileCompactionTask.java | 7 +++--
.../InsertionCrossSpaceCompactionRecoverTest.java | 33 +++++++++++++---------
3 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 77627579252..434c42be5ce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
@@ -443,14 +444,16 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
for (int i = 0; i < filesView.renamedTargetFiles.size(); i++) {
TsFileResource oldFile = filesView.skippedSourceFiles.get(i);
TsFileResource newFile = filesView.renamedTargetFiles.get(i);
- Files.createLink(newFile.getTsFile().toPath(), oldFile.getTsFile().toPath());
- Files.createLink(
+ FileUtils.createLink(newFile.getTsFile().toPath(), oldFile.getTsFile().toPath(), true);
+ FileUtils.createLink(
new File(newFile.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
- new File(oldFile.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).toPath());
+ new File(oldFile.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
+ true);
if (oldFile.modFileExists()) {
- Files.createLink(
+ FileUtils.createLink(
new File(newFile.getTsFilePath() + ModificationFile.FILE_SUFFIX).toPath(),
- new File(oldFile.getTsFilePath() + ModificationFile.FILE_SUFFIX).toPath());
+ new File(oldFile.getTsFilePath() + ModificationFile.FILE_SUFFIX).toPath(),
+ true);
}
newFile.deserialize();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
index f43bd7767b2..56915e364dc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.RepairUnsortedFileCompactionPerformer;
@@ -35,7 +36,6 @@ import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.CountDownLatch;
@@ -169,9 +169,10 @@ public class RepairUnsortedFileCompactionTask extends InnerSpaceCompactionTask {
filesView.sourceFilesInCompactionPerformer, filesView.targetFilesInPerformer);
} else {
if (sourceFile.modFileExists()) {
- Files.createLink(
+ FileUtils.createLink(
new File(filesView.targetFilesInPerformer.get(0).getModFile().getFilePath()).toPath(),
- new File(sourceFile.getModFile().getFilePath()).toPath());
+ new File(sourceFile.getModFile().getFilePath()).toPath(),
+ true);
}
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
index 40fdc2d5cd8..06566613b6c 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.storageengine.dataregion.compaction.cross;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.exception.MergeException;
@@ -54,7 +55,6 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -145,10 +145,11 @@ public class InsertionCrossSpaceCompactionRecoverTest extends AbstractCompaction
File sourceTsFile = unseqResource1.getTsFile();
File targetTsFile = targetFile.getTsFile();
- Files.createLink(targetTsFile.toPath(), sourceTsFile.toPath());
- Files.createLink(
+ FileUtils.createLink(targetTsFile.toPath(), sourceTsFile.toPath(), true);
+ FileUtils.createLink(
new File(targetTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
+ true);
}
// recover compaction, all source file should exist and target file should be deleted
@@ -231,14 +232,16 @@ public class InsertionCrossSpaceCompactionRecoverTest extends AbstractCompaction
File sourceTsFile = unseqResource1.getTsFile();
File targetTsFile = targetFile.getTsFile();
- Files.createLink(targetTsFile.toPath(), sourceTsFile.toPath());
- Files.createLink(
+ FileUtils.createLink(targetTsFile.toPath(), sourceTsFile.toPath(), true);
+ FileUtils.createLink(
new File(targetTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
+ true);
if (unseqResource1.getModFile().exists()) {
- Files.createLink(
+ FileUtils.createLink(
new File(targetTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
+ true);
}
}
@@ -325,14 +328,16 @@ public class InsertionCrossSpaceCompactionRecoverTest extends AbstractCompaction
File sourceTsFile = unseqResource1.getTsFile();
File targetTsFile = targetFile.getTsFile();
- Files.createLink(targetTsFile.toPath(), sourceTsFile.toPath());
- Files.createLink(
+ FileUtils.createLink(targetTsFile.toPath(), sourceTsFile.toPath(), true);
+ FileUtils.createLink(
new File(targetTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + TsFileResource.RESOURCE_SUFFIX).toPath(),
+ true);
if (unseqResource1.getModFile().exists()) {
- Files.createLink(
+ FileUtils.createLink(
new File(targetTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
- new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath());
+ new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
+ true);
}
}