You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/08/14 06:42:41 UTC
[iotdb] branch master updated: fix inaccurate statistics for monitoring the number of unseq Tsfile on cross compaction (#10651)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ccd73310fac fix inaccurate statistics for monitoring the number of unseq Tsfile on cross compaction (#10651)
ccd73310fac is described below
commit ccd73310facc5b0d653a93e4f9c5fef591d2f081
Author: Zhijia Cao <ca...@126.com>
AuthorDate: Mon Aug 14 14:42:36 2023 +0800
fix inaccurate statistics for monitoring the number of unseq Tsfile on cross compaction (#10651)
---
.../exception/CompactionExceptionHandler.java | 11 ++---
.../execute/task/CrossSpaceCompactionTask.java | 23 +---------
.../execute/task/InnerSpaceCompactionTask.java | 14 +-----
.../compaction/execute/utils/CompactionUtils.java | 53 ++++++++++++++++++++++
4 files changed, 59 insertions(+), 42 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
index bb4397cec43..12039324339 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
@@ -226,14 +226,9 @@ public class CompactionExceptionHandler {
return false;
}
- // delete source files
- for (TsFileResource resource : sourceSeqResourceList) {
- resource.remove();
- }
- for (TsFileResource resource : sourceUnseqResourceList) {
- resource.remove();
- }
-
+ // delete sources file
+ CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(
+ sourceSeqResourceList, sourceUnseqResourceList);
// delete compaction mods files
CompactionUtils.deleteCompactionModsFile(sourceSeqResourceList, sourceUnseqResourceList);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
index 5b41382286f..c2d88d7cb11 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -213,14 +213,8 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
}
}
- long[] sequenceFileSize = deleteOldFiles(selectedSequenceFiles);
- List<String> fileNames = new ArrayList<>(selectedSequenceFiles.size());
- selectedSequenceFiles.forEach(x -> fileNames.add(x.getTsFile().getName()));
- FileMetrics.getInstance().deleteFile(sequenceFileSize, true, fileNames);
- fileNames.clear();
- selectedUnsequenceFiles.forEach(x -> fileNames.add(x.getTsFile().getName()));
- long[] unsequenceFileSize = deleteOldFiles(selectedUnsequenceFiles);
- FileMetrics.getInstance().deleteFile(unsequenceFileSize, false, fileNames);
+ CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(
+ selectedSequenceFiles, selectedUnsequenceFiles);
CompactionUtils.deleteCompactionModsFile(selectedSequenceFiles, selectedUnsequenceFiles);
for (TsFileResource targetResource : targetTsfileResourceList) {
@@ -356,19 +350,6 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
return equalsOtherTask((CrossSpaceCompactionTask) other);
}
- private long[] deleteOldFiles(List<TsFileResource> tsFileResourceList) {
- long[] size = new long[tsFileResourceList.size()];
- for (int i = 0, length = tsFileResourceList.size(); i < length; ++i) {
- TsFileResource tsFileResource = tsFileResourceList.get(i);
- size[i] = tsFileResource.getTsFileSize();
- tsFileResource.remove();
- LOGGER.info(
- "[CrossSpaceCompaction] Delete TsFile :{}.",
- tsFileResource.getTsFile().getAbsolutePath());
- }
- return size;
- }
-
private void releaseReadAndLockWrite(List<TsFileResource> tsFileResourceList) {
for (TsFileResource tsFileResource : tsFileResourceList) {
tsFileResource.readUnlock();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 007dbf7e8ac..4e8693f2850 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -231,13 +231,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
"{}-{} [Compaction] compaction finish, start to delete old files",
storageGroupName,
dataRegionId);
- // delete the old files
- long[] sizeList = new long[selectedTsFileResourceList.size()];
- for (int i = 0, size = selectedTsFileResourceList.size(); i < size; ++i) {
- sizeList[i] = selectedTsFileResourceList.get(i).getTsFileSize();
- }
- CompactionUtils.deleteTsFilesInDisk(
- selectedTsFileResourceList, storageGroupName + "-" + dataRegionId);
+ CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(selectedTsFileResourceList);
CompactionUtils.deleteModificationForSourceFile(
selectedTsFileResourceList, storageGroupName + "-" + dataRegionId);
@@ -255,12 +249,6 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
// target resource is empty after compaction, then delete it
targetTsFileResource.remove();
}
- List<String> fileNames = new ArrayList<>();
- for (TsFileResource resource : selectedTsFileResourceList) {
- fileNames.add(resource.getTsFile().getName());
- }
- FileMetrics.getInstance().deleteFile(sizeList, sequence, fileNames);
-
CompactionMetrics.getInstance().recordSummaryInfo(summary);
double costTime = (System.currentTimeMillis() - startTime) / 1000.0d;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index a5160b7a3bf..ea7d41b27d1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -436,4 +436,57 @@ public class CompactionUtils {
}
return true;
}
+
+ public static void deleteSourceTsFileAndUpdateFileMetrics(
+ List<TsFileResource> sourceSeqResourceList, List<TsFileResource> sourceUnseqResourceList) {
+ // delete seq file
+ deleteSourceTsFileAndUpdateFileMetrics(sourceSeqResourceList);
+
+ // delete unSeq file
+ long[] unSequenceFileSize = new long[sourceUnseqResourceList.size()];
+ List<String> unSequenceFileNames = new ArrayList<>();
+ boolean removeSuccess = true;
+ for (int i = 0; i < sourceUnseqResourceList.size(); i++) {
+ TsFileResource tsFileResource = sourceUnseqResourceList.get(i);
+ if (!tsFileResource.remove()) {
+ removeSuccess = false;
+ logger.warn(
+ "[Compaction] delete unSequence file failed,file path is {}",
+ tsFileResource.getTsFile().getAbsolutePath());
+ } else {
+ logger.info(
+ "[Compaction] delete unSequence file :{}",
+ tsFileResource.getTsFile().getAbsolutePath());
+ unSequenceFileSize[i] = tsFileResource.getTsFileSize();
+ unSequenceFileNames.add(tsFileResource.getTsFile().getName());
+ }
+ }
+ if (removeSuccess) {
+ FileMetrics.getInstance().deleteFile(unSequenceFileSize, false, unSequenceFileNames);
+ }
+ }
+
+ public static void deleteSourceTsFileAndUpdateFileMetrics(
+ List<TsFileResource> sourceSeqResourceList) {
+ long[] sequenceFileSize = new long[sourceSeqResourceList.size()];
+ List<String> sequenceFileNames = new ArrayList<>();
+ boolean removeSuccess = true;
+ for (int i = 0; i < sourceSeqResourceList.size(); i++) {
+ TsFileResource tsFileResource = sourceSeqResourceList.get(i);
+ if (!tsFileResource.remove()) {
+ removeSuccess = false;
+ logger.warn(
+ "[Compaction] delete sequence file failed,file path is {}",
+ tsFileResource.getTsFile().getAbsolutePath());
+ } else {
+ logger.info(
+ "[Compaction] delete sequence file :{}", tsFileResource.getTsFile().getAbsolutePath());
+ sequenceFileSize[i] = tsFileResource.getTsFileSize();
+ sequenceFileNames.add(tsFileResource.getTsFile().getName());
+ }
+ }
+ if (removeSuccess) {
+ FileMetrics.getInstance().deleteFile(sequenceFileSize, true, sequenceFileNames);
+ }
+ }
}