You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/03/17 08:15:32 UTC
[iotdb] branch rel/1.1 updated: [To rel/1.1][IOTDB-5651] Fix compaction UT
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new 5397660336 [To rel/1.1][IOTDB-5651] Fix compaction UT
5397660336 is described below
commit 53976603365b17993f7690e947283d42963edf34
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Fri Mar 17 16:15:24 2023 +0800
[To rel/1.1][IOTDB-5651] Fix compaction UT
---
.../execute/task/CrossSpaceCompactionTask.java | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 264 ++++++++++-----------
2 files changed, 135 insertions(+), 136 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 887b930406..e29c083c70 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -213,10 +213,6 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
long unsequenceFileSize = deleteOldFiles(selectedUnsequenceFiles);
CompactionUtils.deleteCompactionModsFile(selectedSequenceFiles, selectedUnsequenceFiles);
- if (logFile.exists()) {
- FileUtils.delete(logFile);
- }
-
// update the metrics finally in case of any exception occurs
for (TsFileResource targetResource : targetTsfileResourceList) {
if (!targetResource.isDeleted()) {
@@ -246,6 +242,9 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
(selectedSeqFileSize + selectedUnseqFileSize) / 1024 / 1024 / costTime,
summary);
}
+ if (logFile.exists()) {
+ FileUtils.delete(logFile);
+ }
} catch (Throwable throwable) {
// catch throwable to handle OOM errors
if (!(throwable instanceof InterruptedException)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
index a9f7aca321..4c871843fa 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -59,6 +59,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
protected int sumOfCompactionCount;
protected long maxFileVersion;
protected int maxCompactionCount;
+ private File logFile;
protected TsFileResourceList tsFileResourceList;
protected List<TsFileResource> targetTsFileList;
@@ -116,153 +117,152 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
sequence ? "Sequence" : "Unsequence",
selectedTsFileResourceList.size(),
selectedFileSize / 1024 / 1024);
+
try {
targetTsFileResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(
selectedTsFileResourceList, sequence);
- } catch (IOException e) {
- LOGGER.error("Failed to get target file for {}", selectedTsFileResourceList, e);
- return;
- }
- File logFile =
- new File(
- dataDirectory
- + File.separator
- + targetTsFileResource.getTsFile().getName()
- + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX);
- try (CompactionLogger compactionLogger = new CompactionLogger(logFile)) {
- // Here is tmpTargetFile, which is xxx.target
- targetTsFileList = new ArrayList<>(Collections.singletonList(targetTsFileResource));
- compactionLogger.logFiles(selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES);
- compactionLogger.logFiles(targetTsFileList, CompactionLogger.STR_TARGET_FILES);
-
- LOGGER.info(
- "{}-{} [Compaction] compaction with {}",
- storageGroupName,
- dataRegionId,
- selectedTsFileResourceList);
-
- // carry out the compaction
- performer.setSourceFiles(selectedTsFileResourceList);
- // As elements in targetFiles may be removed in ReadPointCompactionPerformer, we should use a
- // mutable list instead of Collections.singletonList()
- performer.setTargetFiles(targetTsFileList);
- performer.setSummary(summary);
- performer.perform();
-
- CompactionUtils.moveTargetFile(targetTsFileList, true, storageGroupName + "-" + dataRegionId);
-
- LOGGER.info(
- "{}-{} [InnerSpaceCompactionTask] start to rename mods file",
- storageGroupName,
- dataRegionId);
- CompactionUtils.combineModsInInnerCompaction(
- selectedTsFileResourceList, targetTsFileResource);
-
- if (Thread.currentThread().isInterrupted() || summary.isCancel()) {
- throw new InterruptedException(
- String.format("%s-%s [Compaction] abort", storageGroupName, dataRegionId));
- }
+ logFile =
+ new File(
+ dataDirectory
+ + File.separator
+ + targetTsFileResource.getTsFile().getName()
+ + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX);
+ try (CompactionLogger compactionLogger = new CompactionLogger(logFile)) {
+ // Here is tmpTargetFile, which is xxx.target
+ targetTsFileList = new ArrayList<>(Collections.singletonList(targetTsFileResource));
+ compactionLogger.logFiles(selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES);
+ compactionLogger.logFiles(targetTsFileList, CompactionLogger.STR_TARGET_FILES);
+
+ LOGGER.info(
+ "{}-{} [Compaction] compaction with {}",
+ storageGroupName,
+ dataRegionId,
+ selectedTsFileResourceList);
+
+ // carry out the compaction
+ performer.setSourceFiles(selectedTsFileResourceList);
+ // As elements in targetFiles may be removed in ReadPointCompactionPerformer, we should use
+ // a
+ // mutable list instead of Collections.singletonList()
+ performer.setTargetFiles(targetTsFileList);
+ performer.setSummary(summary);
+ performer.perform();
+
+ CompactionUtils.moveTargetFile(
+ targetTsFileList, true, storageGroupName + "-" + dataRegionId);
+
+ LOGGER.info(
+ "{}-{} [InnerSpaceCompactionTask] start to rename mods file",
+ storageGroupName,
+ dataRegionId);
+ CompactionUtils.combineModsInInnerCompaction(
+ selectedTsFileResourceList, targetTsFileResource);
- // replace the old files with new file, the new is in same position as the old
- if (sequence) {
- tsFileManager.replace(
- selectedTsFileResourceList,
- Collections.emptyList(),
- targetTsFileList,
- timePartition,
- true);
- } else {
- tsFileManager.replace(
- Collections.emptyList(),
- selectedTsFileResourceList,
- targetTsFileList,
- timePartition,
- false);
- }
+ if (Thread.currentThread().isInterrupted() || summary.isCancel()) {
+ throw new InterruptedException(
+ String.format("%s-%s [Compaction] abort", storageGroupName, dataRegionId));
+ }
- if (targetTsFileResource.isDeleted()) {
- compactionLogger.logFile(targetTsFileResource, CompactionLogger.STR_DELETED_TARGET_FILES);
- }
+ // replace the old files with new file, the new is in same position as the old
+ if (sequence) {
+ tsFileManager.replace(
+ selectedTsFileResourceList,
+ Collections.emptyList(),
+ targetTsFileList,
+ timePartition,
+ true);
+ } else {
+ tsFileManager.replace(
+ Collections.emptyList(),
+ selectedTsFileResourceList,
+ targetTsFileList,
+ timePartition,
+ false);
+ }
- if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation()
- && !CompactionUtils.validateTsFileResources(
- tsFileManager, storageGroupName, timePartition)) {
- LOGGER.error(
- "Failed to pass compaction validation, source files is: {}, target files is {}",
- selectedTsFileResourceList,
- targetTsFileList);
- throw new RuntimeException("Failed to pass compaction validation");
- }
+ if (targetTsFileResource.isDeleted()) {
+ compactionLogger.logFile(targetTsFileResource, CompactionLogger.STR_DELETED_TARGET_FILES);
+ }
- LOGGER.info(
- "{}-{} [Compaction] Compacted target files, try to get the write lock of source files",
- storageGroupName,
- dataRegionId);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation()
+ && !CompactionUtils.validateTsFileResources(
+ tsFileManager, storageGroupName, timePartition)) {
+ LOGGER.error(
+ "Failed to pass compaction validation, source files is: {}, target files is {}",
+ selectedTsFileResourceList,
+ targetTsFileList);
+ throw new RuntimeException("Failed to pass compaction validation");
+ }
- // release the read lock of all source files, and get the write lock of them to delete them
- for (int i = 0; i < selectedTsFileResourceList.size(); ++i) {
- selectedTsFileResourceList.get(i).readUnlock();
- isHoldingReadLock[i] = false;
- selectedTsFileResourceList.get(i).writeLock();
- isHoldingWriteLock[i] = true;
- }
+ LOGGER.info(
+ "{}-{} [Compaction] Compacted target files, try to get the write lock of source files",
+ storageGroupName,
+ dataRegionId);
+
+ // release the read lock of all source files, and get the write lock of them to delete them
+ for (int i = 0; i < selectedTsFileResourceList.size(); ++i) {
+ selectedTsFileResourceList.get(i).readUnlock();
+ isHoldingReadLock[i] = false;
+ selectedTsFileResourceList.get(i).writeLock();
+ isHoldingWriteLock[i] = true;
+ }
- if (targetTsFileResource.getTsFile().exists()
- && targetTsFileResource.getTsFile().length()
- < TSFileConfig.MAGIC_STRING.getBytes().length * 2L + Byte.BYTES) {
- // the file size is smaller than magic string and version number
- throw new TsFileNotCompleteException(
- String.format(
- "target file %s is smaller than magic string and version number size",
- targetTsFileResource));
- }
+ if (targetTsFileResource.getTsFile().exists()
+ && targetTsFileResource.getTsFile().length()
+ < TSFileConfig.MAGIC_STRING.getBytes().length * 2L + Byte.BYTES) {
+ // the file size is smaller than magic string and version number
+ throw new TsFileNotCompleteException(
+ String.format(
+ "target file %s is smaller than magic string and version number size",
+ targetTsFileResource));
+ }
- LOGGER.info(
- "{}-{} [Compaction] compaction finish, start to delete old files",
- storageGroupName,
- dataRegionId);
- // delete the old files
- long totalSizeOfDeletedFile = 0L;
- for (TsFileResource resource : selectedTsFileResourceList) {
- totalSizeOfDeletedFile += resource.getTsFileSize();
- }
- CompactionUtils.deleteTsFilesInDisk(
- selectedTsFileResourceList, storageGroupName + "-" + dataRegionId);
- CompactionUtils.deleteModificationForSourceFile(
- selectedTsFileResourceList, storageGroupName + "-" + dataRegionId);
+ LOGGER.info(
+ "{}-{} [Compaction] compaction finish, start to delete old files",
+ storageGroupName,
+ dataRegionId);
+ // delete the old files
+ long totalSizeOfDeletedFile = 0L;
+ for (TsFileResource resource : selectedTsFileResourceList) {
+ totalSizeOfDeletedFile += resource.getTsFileSize();
+ }
+ CompactionUtils.deleteTsFilesInDisk(
+ selectedTsFileResourceList, storageGroupName + "-" + dataRegionId);
+ CompactionUtils.deleteModificationForSourceFile(
+ selectedTsFileResourceList, storageGroupName + "-" + dataRegionId);
+
+ // inner space compaction task has only one target file
+ if (!targetTsFileResource.isDeleted()) {
+ TsFileMetricManager.getInstance()
+ .addFile(targetTsFileResource.getTsFile().length(), sequence);
+
+ // set target resource to CLOSED, so that it can be selected to compact
+ targetTsFileResource.setStatus(TsFileResourceStatus.CLOSED);
+ } else {
+ // target resource is empty after compaction, then delete it
+ targetTsFileResource.remove();
+ }
+ TsFileMetricManager.getInstance()
+ .deleteFile(totalSizeOfDeletedFile, sequence, selectedTsFileResourceList.size());
+
+ CompactionMetricsManager.getInstance().updateSummary(summary);
+ double costTime = (System.currentTimeMillis() - startTime) / 1000.0d;
+ LOGGER.info(
+ "{}-{} [Compaction] {} InnerSpaceCompaction task finishes successfully, target file is {},"
+ + "time cost is {} s, compaction speed is {} MB/s, {}",
+ storageGroupName,
+ dataRegionId,
+ sequence ? "Sequence" : "Unsequence",
+ targetTsFileResource.getTsFile().getName(),
+ costTime,
+ selectedFileSize / 1024.0d / 1024.0d / costTime,
+ summary);
+ }
if (logFile.exists()) {
FileUtils.delete(logFile);
}
-
- // inner space compaction task has only one target file
- if (!targetTsFileResource.isDeleted()) {
- TsFileMetricManager.getInstance()
- .addFile(targetTsFileResource.getTsFile().length(), sequence);
-
- // set target resource to CLOSED, so that it can be selected to compact
- targetTsFileResource.setStatus(TsFileResourceStatus.CLOSED);
- } else {
- // target resource is empty after compaction, then delete it
- targetTsFileResource.remove();
- }
- TsFileMetricManager.getInstance()
- .deleteFile(totalSizeOfDeletedFile, sequence, selectedTsFileResourceList.size());
-
- CompactionMetricsManager.getInstance().updateSummary(summary);
-
- double costTime = (System.currentTimeMillis() - startTime) / 1000.0d;
- LOGGER.info(
- "{}-{} [Compaction] {} InnerSpaceCompaction task finishes successfully, target file is {},"
- + "time cost is {} s, compaction speed is {} MB/s, {}",
- storageGroupName,
- dataRegionId,
- sequence ? "Sequence" : "Unsequence",
- targetTsFileResource.getTsFile().getName(),
- costTime,
- selectedFileSize / 1024.0d / 1024.0d / costTime,
- summary);
} catch (Throwable throwable) {
// catch throwable to handle OOM errors
if (!(throwable instanceof InterruptedException)) {