You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/10/19 07:02:22 UTC
[iotdb] branch compaction_recover_logger_1017 updated: add UT
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch compaction_recover_logger_1017
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/compaction_recover_logger_1017 by this push:
new cf2ca728803 add UT
cf2ca728803 is described below
commit cf2ca7288030e6f686ae1536bcaa9e5bac194ec0
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu Oct 19 15:02:10 2023 +0800
add UT
---
.../execute/task/AbstractCompactionTask.java | 4 +-
.../execute/task/CrossSpaceCompactionTask.java | 1 -
.../execute/task/InnerSpaceCompactionTask.java | 1 -
...eCrossSpaceCompactionRecoverCompatibleTest.java | 413 ---------------
.../SizeTieredCompactionRecoverCompatibleTest.java | 276 ----------
.../recover/SizeTieredCompactionRecoverTest.java | 586 ++++++++++++++-------
6 files changed, 411 insertions(+), 870 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index d7d5c36687f..a67c053e9c6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -289,8 +289,8 @@ public abstract class AbstractCompactionTask {
protected void deleteCompactionModsFile(List<TsFileResource> tsFileResourceList)
throws IOException {
- for (TsFileResource seqFile : tsFileResourceList) {
- ModificationFile modificationFile = seqFile.getCompactionModFile();
+ for (TsFileResource tsFile : tsFileResourceList) {
+ ModificationFile modificationFile = tsFile.getCompactionModFile();
if (modificationFile.exists()) {
modificationFile.remove();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
index f03ab79f12c..eb515f3ae03 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -359,7 +359,6 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
throw new CompactionRecoverException("source files cannot be deleted successfully");
}
if (recoverMemoryStatus) {
- // 这个地方的统计可能是不准确的
FileMetrics.getInstance().deleteTsFile(true, selectedSequenceFiles);
FileMetrics.getInstance().deleteTsFile(true, selectedUnsequenceFiles);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 180ba500877..48a5b25c406 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -380,7 +380,6 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
throw new CompactionRecoverException("source files cannot be deleted successfully");
}
if (recoverMemoryStatus) {
- // 这个地方的统计可能是不准确的
FileMetrics.getInstance().deleteTsFile(true, selectedTsFileResourceList);
}
deleteCompactionModsFile(selectedTsFileResourceList);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java
deleted file mode 100644
index ed90707cd3c..00000000000
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.storageengine.dataregion.compaction.recover;
-
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverTask;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionFileGeneratorUtils;
-import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
-import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import com.google.common.io.Files;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
-
-public class RewriteCrossSpaceCompactionRecoverCompatibleTest extends AbstractCompactionTest {
- private final String oldThreadName = Thread.currentThread().getName();
-
- @Override
- @Before
- public void setUp()
- throws IOException, MetadataException, WriteProcessException, InterruptedException {
- super.setUp();
- Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
- }
-
- @Override
- @After
- public void tearDown() throws StorageEngineException, IOException {
- super.tearDown();
- Thread.currentThread().setName(oldThreadName);
- }
-
- @Test
- public void testCompatibleWithAllSourceFilesExistWithFileInfo() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- createFiles(3, 2, 3, 50, 0, 0, 50, 50, false, false);
- registerTimeseriesInMManger(2, 3, false);
- List<TsFileResource> tmpTargetResources = new ArrayList<>();
- for (TsFileResource resource : seqResources) {
- File mergeFile =
- new File(
- resource.getTsFilePath() + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX_FROM_OLD);
- mergeFile.createNewFile();
- tmpTargetResources.add(new TsFileResource(mergeFile));
- }
-
- File logFile =
- new File(SEQ_DIRS.getPath(), CompactionLogger.CROSS_COMPACTION_LOG_NAME_FROM_OLD);
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- logWriter.write("seqFiles\n");
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(
- String.format(
- COMPACTION_TEST_SG + " 0 0 %s true\n", tsFileResource.getTsFile().getName()));
- }
- logWriter.write("unseqFiles\n");
- for (TsFileResource tsFileResource : unseqResources) {
- logWriter.write(
- String.format(
- COMPACTION_TEST_SG + " 0 0 %s false\n", tsFileResource.getTsFile().getName()));
- }
- logWriter.close();
-
- for (int i = 0; i < seqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false);
- }
- for (int i = 0; i < unseqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s1",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false);
- }
- ModificationFile mergeMods =
- new ModificationFile(
- SEQ_DIRS + File.separator + IoTDBConstant.COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD);
- mergeMods.write(new Deletion(new PartialPath("root.d1.s1"), 100, 0, 100));
- mergeMods.close();
-
- TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath());
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, false)
- .doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
- for (TsFileResource resource : unseqResources) {
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
- for (TsFileResource resource : tmpTargetResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- }
- for (TsFileResource resource :
- CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources)) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- }
-
- Assert.assertFalse(logFile.exists());
- Assert.assertFalse(mergeMods.exists());
- }
-
- @Test
- public void testCompatibleWithSomeSourceFilesLostWithFileInfo() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- createFiles(3, 2, 3, 50, 0, 0, 50, 50, false, false);
- registerTimeseriesInMManger(2, 3, false);
- List<TsFileResource> tmpTargetResources = new ArrayList<>();
- for (TsFileResource resource : seqResources) {
- File mergeFile =
- new File(
- resource.getTsFilePath() + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX_FROM_OLD);
- Files.copy(resource.getTsFile(), mergeFile);
- tmpTargetResources.add(new TsFileResource(mergeFile));
- }
-
- File logFile =
- new File(SEQ_DIRS.getPath(), CompactionLogger.CROSS_COMPACTION_LOG_NAME_FROM_OLD);
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- logWriter.write("seqFiles\n");
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(
- String.format(
- COMPACTION_TEST_SG + " 0 0 %s true\n", tsFileResource.getTsFile().getName()));
- }
- logWriter.write("unseqFiles\n");
- for (TsFileResource tsFileResource : unseqResources) {
- logWriter.write(
- String.format(
- COMPACTION_TEST_SG + " 0 0 %s false\n", tsFileResource.getTsFile().getName()));
- }
- logWriter.close();
-
- // First seq files lost
- seqResources.get(0).remove();
-
- for (int i = 0; i < seqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false);
- }
- for (int i = 0; i < unseqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s1",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false);
- }
- ModificationFile mergeMods =
- new ModificationFile(
- SEQ_DIRS + File.separator + IoTDBConstant.COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD);
- mergeMods.write(new Deletion(new PartialPath("root.d1.s1"), 100, 0, 100));
- mergeMods.close();
-
- TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath());
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, false)
- .doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- Assert.assertFalse(resource.getModFile().exists());
- }
- for (TsFileResource resource : unseqResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- Assert.assertFalse(resource.getModFile().exists());
- }
- for (TsFileResource resource : tmpTargetResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- }
- for (TsFileResource seqResource :
- CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources)) {
- TsFileResource resource =
- new TsFileResource(
- new File(
- seqResource
- .getTsFilePath()
- .replace(
- IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX,
- TsFileConstant.TSFILE_SUFFIX)));
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
-
- Assert.assertFalse(logFile.exists());
- Assert.assertFalse(mergeMods.exists());
- }
-
- @Test
- public void testCompatibleWithAllSourceFilesExistWithFilePath() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- createFiles(3, 2, 3, 50, 0, 0, 50, 50, false, false);
- registerTimeseriesInMManger(2, 3, false);
- List<TsFileResource> tmpTargetResources = new ArrayList<>();
- for (TsFileResource resource : seqResources) {
- File mergeFile =
- new File(
- resource.getTsFilePath() + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX_FROM_OLD);
- mergeFile.createNewFile();
- tmpTargetResources.add(new TsFileResource(mergeFile));
- }
-
- File logFile =
- new File(SEQ_DIRS.getPath(), CompactionLogger.CROSS_COMPACTION_LOG_NAME_FROM_OLD);
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- logWriter.write("seqFiles\n");
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(tsFileResource.getTsFile().getAbsolutePath());
- logWriter.write("\n");
- }
- logWriter.write("unseqFiles\n");
- for (TsFileResource tsFileResource : unseqResources) {
- logWriter.write(tsFileResource.getTsFile().getAbsolutePath());
- logWriter.write("\n");
- }
- logWriter.write(CompactionLogger.STR_MERGE_START_FROM_OLD);
- logWriter.write("\n");
- logWriter.write(seqResources.get(0).getTsFile().getAbsolutePath() + " " + 100);
- logWriter.close();
-
- for (int i = 0; i < seqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false);
- }
- for (int i = 0; i < unseqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s1",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false);
- }
- ModificationFile mergeMods =
- new ModificationFile(
- SEQ_DIRS + File.separator + IoTDBConstant.COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD);
- mergeMods.write(new Deletion(new PartialPath("root.d1.s1"), 100, 0, 100));
- mergeMods.close();
-
- TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath());
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, false)
- .doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
- for (TsFileResource resource : unseqResources) {
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
- for (TsFileResource resource : tmpTargetResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- }
- for (TsFileResource resource :
- CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources)) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- }
-
- Assert.assertFalse(logFile.exists());
- Assert.assertFalse(mergeMods.exists());
- }
-
- @Test
- public void testCompatibleWithSomeSourceFilesLostWithFilePath() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- createFiles(3, 2, 3, 50, 0, 0, 50, 50, false, false);
- registerTimeseriesInMManger(2, 3, false);
- List<TsFileResource> tmpTargetResources = new ArrayList<>();
- for (TsFileResource resource : seqResources) {
- File mergeFile =
- new File(
- resource.getTsFilePath() + IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX_FROM_OLD);
- Files.copy(resource.getTsFile(), mergeFile);
- tmpTargetResources.add(new TsFileResource(mergeFile));
- }
-
- File logFile =
- new File(SEQ_DIRS.getPath(), CompactionLogger.CROSS_COMPACTION_LOG_NAME_FROM_OLD);
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- logWriter.write("seqFiles\n");
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(tsFileResource.getTsFile().getAbsolutePath());
- logWriter.write("\n");
- }
- logWriter.write("unseqFiles\n");
- for (TsFileResource tsFileResource : unseqResources) {
- logWriter.write(tsFileResource.getTsFile().getAbsolutePath());
- logWriter.write("\n");
- }
- logWriter.write(CompactionLogger.STR_MERGE_START_FROM_OLD);
- logWriter.write("\n");
- logWriter.write(seqResources.get(0).getTsFile().getAbsolutePath() + " " + 100);
- logWriter.close();
-
- // First seq files lost
- seqResources.get(0).remove();
-
- for (int i = 0; i < seqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false);
- }
- for (int i = 0; i < unseqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s1",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false);
- }
- ModificationFile mergeMods =
- new ModificationFile(
- SEQ_DIRS + File.separator + IoTDBConstant.COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD);
- mergeMods.write(new Deletion(new PartialPath("root.d1.s1"), 100, 0, 100));
- mergeMods.close();
-
- TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath());
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, false)
- .doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- Assert.assertFalse(resource.getModFile().exists());
- }
- for (TsFileResource resource : unseqResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- Assert.assertFalse(resource.getModFile().exists());
- }
- for (TsFileResource resource : tmpTargetResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- }
- for (TsFileResource seqResource :
- CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources)) {
- TsFileResource resource =
- new TsFileResource(
- new File(
- seqResource
- .getTsFilePath()
- .replace(
- IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX,
- TsFileConstant.TSFILE_SUFFIX)));
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
-
- Assert.assertFalse(logFile.exists());
- Assert.assertFalse(mergeMods.exists());
- }
-}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
deleted file mode 100644
index 35793690f82..00000000000
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.compaction.recover;
-
-import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverTask;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionFileGeneratorUtils;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
-
-/** This test is used to test the compatibility of compaction recovery with 0.12. */
-public class SizeTieredCompactionRecoverCompatibleTest extends AbstractCompactionTest {
- @Override
- @Before
- public void setUp()
- throws IOException, MetadataException, WriteProcessException, InterruptedException {
- super.setUp();
- }
-
- @Override
- @After
- public void tearDown() throws StorageEngineException, IOException {
- super.tearDown();
- }
-
- @Test
- public void testCompatibleWithAllSourceFilesExistWithFilePath() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- registerTimeseriesInMManger(2, 3, false);
- TsFileResource targetResource =
- TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true);
- ICompactionPerformer performer = new ReadChunkCompactionPerformer(seqResources, targetResource);
- performer.setSummary(new CompactionTaskSummary());
- performer.perform();
- RandomAccessFile targetFile = new RandomAccessFile(targetResource.getTsFile(), "rw");
- long fileLength = targetFile.length();
- targetFile.getChannel().truncate(fileLength - 20);
- targetFile.close();
-
- for (int i = 0; i < seqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false);
- }
-
- File logFile =
- new File(
- targetResource.getTsFile().getParent(),
- COMPACTION_TEST_SG + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX_FROM_OLD);
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(
- String.format(
- "info-source\n%s 0 0 %s sequence\n",
- COMPACTION_TEST_SG, tsFileResource.getTsFile().getName()));
- }
- logWriter.write("sequence\n");
- logWriter.write(
- String.format(
- "info-target\n%s 0 0 %s sequence\n",
- COMPACTION_TEST_SG, targetResource.getTsFile().getName()));
- logWriter.close();
-
- TsFileManager tsFileManager =
- new TsFileManager(COMPACTION_TEST_SG, "0", targetResource.getTsFile().getParent());
- tsFileManager.addAll(seqResources, true);
- CompactionRecoverTask recoverTask =
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, true);
- recoverTask.doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertTrue(resource.getTsFile().exists());
- Assert.assertTrue(resource.resourceFileExists());
- Assert.assertTrue(resource.getModFile().exists());
- }
-
- Assert.assertFalse(targetResource.getTsFile().exists());
- Assert.assertFalse(targetResource.resourceFileExists());
- Assert.assertFalse(targetResource.getModFile().exists());
- Assert.assertFalse(logFile.exists());
- }
-
- @Test
- public void testCompatibleWithSomeSourceFilesLostWithFilePath() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- registerTimeseriesInMManger(2, 3, false);
- TsFileResource targetResource =
- TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true);
- ICompactionPerformer performer = new ReadChunkCompactionPerformer(seqResources, targetResource);
- performer.setSummary(new CompactionTaskSummary());
- performer.perform();
- CompactionUtils.moveTargetFile(
- Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
-
- // first source file does not exist
- seqResources.get(0).remove();
-
- for (int i = 0; i < seqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false);
- }
-
- File logFile =
- new File(
- targetResource.getTsFile().getParent(),
- COMPACTION_TEST_SG + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX_FROM_OLD);
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(
- String.format(
- "info-source\n%s 0 0 %s sequence\n",
- COMPACTION_TEST_SG, tsFileResource.getTsFile().getName()));
- }
- logWriter.write("sequence\n");
- logWriter.write(
- String.format(
- "info-target\n%s 0 0 %s sequence\n",
- COMPACTION_TEST_SG, targetResource.getTsFile().getName()));
- logWriter.close();
-
- TsFileManager tsFileManager =
- new TsFileManager(COMPACTION_TEST_SG, "0", targetResource.getTsFile().getParent());
- tsFileManager.addAll(seqResources, true);
- CompactionRecoverTask recoverTask =
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, true);
- recoverTask.doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- Assert.assertFalse(resource.getModFile().exists());
- }
-
- Assert.assertTrue(targetResource.getTsFile().exists());
- Assert.assertTrue(targetResource.resourceFileExists());
- Assert.assertFalse(logFile.exists());
- }
-
- @Test
- public void testCompatibleWithAllSourceFilesExistWithFileInfo() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
- registerTimeseriesInMManger(2, 3, false);
- TsFileResource targetResource =
- TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true);
- ICompactionPerformer performer = new ReadChunkCompactionPerformer(seqResources, targetResource);
- performer.setSummary(new CompactionTaskSummary());
- performer.perform();
- RandomAccessFile targetFile = new RandomAccessFile(targetResource.getTsFile(), "rw");
- long fileLength = targetFile.length();
- targetFile.getChannel().truncate(fileLength - 20);
- targetFile.close();
-
- File logFile =
- new File(targetResource.getTsFile().getParent(), "root.compactionTest" + ".compaction.log");
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- for (TsFileResource tsFileResource : seqResources) {
- logWriter.write(String.format("source\n%s\n", tsFileResource.getTsFile().getAbsolutePath()));
- }
- logWriter.write("sequence\n");
- logWriter.write(String.format("target\n%s\n", targetResource.getTsFile().getAbsolutePath()));
- logWriter.close();
-
- TsFileManager tsFileManager =
- new TsFileManager("root.compactionTest", "0", targetResource.getTsFile().getParent());
- tsFileManager.addAll(seqResources, true);
- CompactionRecoverTask recoverTask =
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, true);
- recoverTask.doCompaction();
-
- for (TsFileResource resource : seqResources) {
- Assert.assertTrue(resource.getTsFile().exists());
- }
-
- Assert.assertFalse(targetResource.getTsFile().exists());
- Assert.assertFalse(logFile.exists());
- }
-
- @Test
- public void testCompatibleWithSomeSourceFilesLostWithFileInfo() throws Exception {
- createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, false);
- registerTimeseriesInMManger(2, 3, false);
- TsFileResource targetResource =
- TsFileNameGenerator.getInnerCompactionTargetFileResource(unseqResources, true);
- ICompactionPerformer performer = new ReadChunkCompactionPerformer(seqResources, targetResource);
- performer.setSummary(new CompactionTaskSummary());
- performer.perform();
- CompactionUtils.moveTargetFile(
- Collections.singletonList(targetResource), true, "root.compactionTest");
-
- // first source file does not exist
- unseqResources.get(0).remove();
-
- for (int i = 0; i < unseqResources.size(); i++) {
- Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
- deleteMap.put(
- COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
- new Pair(i * 10L, i * 10L + 10));
- CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false);
- }
-
- File logFile =
- new File(targetResource.getTsFile().getParent(), "root.compactionTest" + ".compaction.log");
- BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
- for (TsFileResource tsFileResource : unseqResources) {
- logWriter.write(String.format("source\n%s\n", tsFileResource.getTsFile().getAbsolutePath()));
- }
- logWriter.write("sequence\n");
- logWriter.write(String.format("target\n%s\n", targetResource.getTsFile().getAbsolutePath()));
- logWriter.close();
-
- TsFileManager tsFileManager =
- new TsFileManager("root.compactionTest", "0", targetResource.getTsFile().getParent());
- tsFileManager.addAll(unseqResources, false);
- CompactionRecoverTask recoverTask =
- new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, true);
- recoverTask.doCompaction();
-
- for (TsFileResource resource : unseqResources) {
- Assert.assertFalse(resource.getTsFile().exists());
- Assert.assertFalse(resource.resourceFileExists());
- Assert.assertFalse(resource.getModFile().exists());
- }
-
- Assert.assertTrue(targetResource.getTsFile().exists());
- Assert.assertTrue(targetResource.resourceFileExists());
- Assert.assertFalse(logFile.exists());
- }
-}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverTest.java
index 3d23d55a511..9e009a0d913 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/recover/SizeTieredCompactionRecoverTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.recover;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.path.PartialPath;
@@ -30,9 +31,11 @@ import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.SimpleCompactionLogger;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionFileGeneratorUtils;
@@ -200,8 +203,7 @@ public class SizeTieredCompactionRecoverTest {
*
* @throws Exception
*/
- @Test
- public void testRecoverWithCompleteTargetFileUsingFileInfo() throws Exception {
+ private List<TsFileResource> getSourceFiles() throws IllegalPathException, IOException {
List<TsFileResource> sourceFiles = new ArrayList<>();
for (int i = 0; i < 10; i++) {
Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
@@ -226,11 +228,17 @@ public class SizeTieredCompactionRecoverTest {
fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
sourceFiles.add(tsFileResource);
}
+ return sourceFiles;
+ }
+
+ @Test
+ public void testRecoverWithCompleteTargetFileUsingFileInfo() throws Exception {
+ List<TsFileResource> sourceFiles = getSourceFiles();
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES);
- compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES);
+ compactionLogger.logFile(targetResource, STR_TARGET_FILES);
compactionLogger.close();
performer.setSourceFiles(sourceFiles);
performer.setTargetFiles(Collections.singletonList(targetResource));
@@ -238,6 +246,7 @@ public class SizeTieredCompactionRecoverTest {
performer.perform();
CompactionUtils.moveTargetFile(
Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+
CompactionRecoverTask recoverTask =
new CompactionRecoverTask(
COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true);
@@ -249,6 +258,31 @@ public class SizeTieredCompactionRecoverTest {
Assert.assertFalse(targetResource.getTsFile().exists());
}
+ @Test
+ public void testInnerRecoverWithCompleteTargetFileUsingFileInfo() throws Exception {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+
+ InnerSpaceCompactionTask innerSpaceCompactionTask =
+ new InnerSpaceCompactionTask(COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ innerSpaceCompactionTask.recover();
+ for (TsFileResource resource : sourceFiles) {
+ Assert.assertTrue(resource.getTsFile().exists());
+ }
+ Assert.assertFalse(targetResource.getTsFile().exists());
+ }
+
/**
* Test a compaction task in not finished. The compaction log use file info to record files.
*
@@ -256,30 +290,7 @@ public class SizeTieredCompactionRecoverTest {
*/
@Test
public void testRecoverWithIncompleteTargetFileUsingFileInfo() throws Exception {
- List<TsFileResource> sourceFiles = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- }
+ List<TsFileResource> sourceFiles = getSourceFiles();
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
@@ -307,6 +318,35 @@ public class SizeTieredCompactionRecoverTest {
Assert.assertFalse(targetResource.getTsFile().exists());
}
+ @Test
+ public void testInnerRecoverWithIncompleteTargetFileUsingFileInfo() throws Exception {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true);
+ FileChannel channel = targetStream.getChannel();
+ channel.truncate(targetResource.getTsFile().length() - 100);
+ channel.close();
+ InnerSpaceCompactionTask recoverTask =
+ new InnerSpaceCompactionTask(COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ recoverTask.recover();
+ // all the source file should be deleted
+ for (TsFileResource resource : sourceFiles) {
+ Assert.assertTrue(resource.getTsFile().exists());
+ }
+ Assert.assertFalse(targetResource.getTsFile().exists());
+ }
+
/**
* Test a compaction task in finished. The compaction log use file path to record files. All the
* sources file is still existed.
@@ -315,30 +355,7 @@ public class SizeTieredCompactionRecoverTest {
*/
@Test
public void testRecoverWithCompleteTargetFileUsingFilePath() throws Exception {
- List<TsFileResource> sourceFiles = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- }
+ List<TsFileResource> sourceFiles = getSourceFiles();
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger logger = new CompactionLogger(new File(logFilePath));
@@ -362,6 +379,31 @@ public class SizeTieredCompactionRecoverTest {
Assert.assertFalse(targetResource.getTsFile().exists());
}
+ @Test
+ public void testInnerRecoverWithCompleteTargetFileUsingFilePath() throws Exception {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger logger = new SimpleCompactionLogger(new File(logFilePath));
+ logger.logSourceFiles(sourceFiles);
+ logger.logTargetFile(targetResource);
+ logger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ InnerSpaceCompactionTask innerSpaceCompactionTask =
+ new InnerSpaceCompactionTask(COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ innerSpaceCompactionTask.recover();
+ // all the source file should still exist
+ for (TsFileResource resource : sourceFiles) {
+ Assert.assertTrue(resource.getTsFile().exists());
+ }
+ Assert.assertFalse(targetResource.getTsFile().exists());
+ }
+
/**
* Test a compaction task in not finished. The compaction log use file path to record files.
*
@@ -369,30 +411,7 @@ public class SizeTieredCompactionRecoverTest {
*/
@Test
public void testRecoverWithIncompleteTargetFileUsingFilePath() throws Exception {
- List<TsFileResource> sourceFiles = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- }
+ List<TsFileResource> sourceFiles = getSourceFiles();
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
@@ -420,6 +439,35 @@ public class SizeTieredCompactionRecoverTest {
Assert.assertFalse(targetResource.getTsFile().exists());
}
+ @Test
+ public void testInnerRecoverWithIncompleteTargetFileUsingFilePath() throws Exception {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true);
+ FileChannel channel = targetStream.getChannel();
+ channel.truncate(targetResource.getTsFile().length() - 100);
+ channel.close();
+ InnerSpaceCompactionTask innerSpaceCompactionTask =
+ new InnerSpaceCompactionTask(COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ innerSpaceCompactionTask.recover();
+ // all the source file should be deleted
+ for (TsFileResource resource : sourceFiles) {
+ Assert.assertTrue(resource.getTsFile().exists());
+ }
+ Assert.assertFalse(targetResource.getTsFile().exists());
+ }
+
/**
* Test a compaction task is finished, and the data dirs of the system is change. The compaction
* log use file info to record files.
@@ -429,32 +477,9 @@ public class SizeTieredCompactionRecoverTest {
@Test
public void testRecoverWithCompleteTargetFileUsingFileInfoAndChangingDataDirs() throws Exception {
try {
- List<TsFileResource> sourceFiles = new ArrayList<>();
+ List<TsFileResource> sourceFiles = getSourceFiles();
List<String> sourceFileNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- sourceFileNames.add(tsFileResource.getTsFile().getName());
- }
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
@@ -467,7 +492,6 @@ public class SizeTieredCompactionRecoverTest {
performer.perform();
CompactionUtils.moveTargetFile(
Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
- long sizeOfTargetFile = targetResource.getTsFileSize();
FileUtils.moveDirectory(
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
@@ -519,6 +543,76 @@ public class SizeTieredCompactionRecoverTest {
}
}
+ @Test
+ public void testInnerRecoverWithCompleteTargetFileUsingFileInfoAndChangingDataDirs()
+ throws Exception {
+ try {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ List<String> sourceFileNames = new ArrayList<>();
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ FileUtils.moveDirectory(
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
+ InnerSpaceCompactionTask recoverTask =
+ new InnerSpaceCompactionTask(
+ COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ recoverTask.recover();
+ // all the source files should exist
+ for (String sourceFileName : sourceFileNames) {
+ Assert.assertTrue(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ sourceFileName)
+ .exists());
+ }
+ File targetFileAfterMoved =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ targetResource
+ .getTsFile()
+ .getName()
+ .replace(
+ IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX,
+ TsFileConstant.TSFILE_SUFFIX));
+ Assert.assertFalse(targetFileAfterMoved.exists());
+ } finally {
+ FileUtils.deleteDirectory(new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ }
+ }
+
/**
* Test a compaction task is not finished, and the data dirs of the system is change The
* compaction log use file info to record files.
@@ -529,32 +623,9 @@ public class SizeTieredCompactionRecoverTest {
public void testRecoverWithIncompleteTargetFileUsingFileInfoAndChangingDataDirs()
throws Exception {
try {
- List<TsFileResource> sourceFiles = new ArrayList<>();
+ List<TsFileResource> sourceFiles = getSourceFiles();
List<String> sourceFileNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- sourceFileNames.add(tsFileResource.getTsFile().getName());
- }
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
@@ -617,6 +688,75 @@ public class SizeTieredCompactionRecoverTest {
}
}
+ @Test
+ public void testInnerRecoverWithIncompleteTargetFileUsingFileInfoAndChangingDataDirs()
+ throws Exception {
+ try {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ List<String> sourceFileNames = new ArrayList<>();
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true);
+ FileChannel channel = targetStream.getChannel();
+ channel.truncate(targetResource.getTsFile().length() - 100);
+ channel.close();
+ FileUtils.moveDirectory(
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
+ InnerSpaceCompactionTask recoverTask =
+ new InnerSpaceCompactionTask(
+ COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ recoverTask.recover();
+ // all the source file should be deleted
+ for (String sourceFileName : sourceFileNames) {
+ Assert.assertTrue(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ sourceFileName)
+ .exists());
+ }
+ Assert.assertFalse(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ targetResource.getTsFile().getName())
+ .exists());
+ } finally {
+ FileUtils.deleteDirectory(new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ }
+ }
+
/**
* Test a compaction task is finished, and the data dirs of the system is change. The compaction
* log use file path to record files.
@@ -626,32 +766,9 @@ public class SizeTieredCompactionRecoverTest {
@Test
public void testRecoverWithCompleteTargetFileUsingFilePathAndChangingDataDirs() throws Exception {
try {
- List<TsFileResource> sourceFiles = new ArrayList<>();
+ List<TsFileResource> sourceFiles = getSourceFiles();
List<String> sourceFileNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- sourceFileNames.add(tsFileResource.getTsFile().getName());
- }
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
@@ -664,7 +781,6 @@ public class SizeTieredCompactionRecoverTest {
performer.perform();
CompactionUtils.moveTargetFile(
Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
- long sizeOfTargetFile = targetResource.getTsFileSize();
FileUtils.moveDirectory(
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
@@ -716,6 +832,76 @@ public class SizeTieredCompactionRecoverTest {
}
}
+ @Test
+ public void testInnerRecoverWithCompleteTargetFileUsingFilePathAndChangingDataDirs()
+ throws Exception {
+ try {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ List<String> sourceFileNames = new ArrayList<>();
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ FileUtils.moveDirectory(
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
+ InnerSpaceCompactionTask recoverTask =
+ new InnerSpaceCompactionTask(
+ COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ recoverTask.recover();
+ // all the source files should exist
+ for (String sourceFileName : sourceFileNames) {
+ Assert.assertTrue(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ sourceFileName)
+ .exists());
+ }
+ File targetFileAfterMoved =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ targetResource
+ .getTsFile()
+ .getName()
+ .replace(
+ IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX,
+ TsFileConstant.TSFILE_SUFFIX));
+ Assert.assertFalse(targetFileAfterMoved.exists());
+ } finally {
+ FileUtils.deleteDirectory(new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ }
+ }
+
/**
* Test a compaction task is not finished, and the data dirs of the system is change. The
* compaction log use file path to record files.
@@ -726,32 +912,9 @@ public class SizeTieredCompactionRecoverTest {
public void testRecoverWithIncompleteTargetFileUsingFilePathAndChangingDataDirs()
throws Exception {
try {
- List<TsFileResource> sourceFiles = new ArrayList<>();
+ List<TsFileResource> sourceFiles = getSourceFiles();
List<String> sourceFileNames = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- Set<String> fullPath = new HashSet<>(Arrays.asList(fullPaths));
- List<List<Long>> chunkPagePointsNum = new ArrayList<>();
- List<Long> pagePointsNum = new ArrayList<>();
- pagePointsNum.add(100L);
- chunkPagePointsNum.add(pagePointsNum);
- TsFileResource tsFileResource =
- new TsFileResource(
- new File(
- SEQ_FILE_DIR
- + File.separator.concat(
- i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + i
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + IoTDBConstant.FILE_NAME_SEPARATOR
- + 0
- + ".tsfile")));
- CompactionFileGeneratorUtils.writeTsFile(
- fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
- sourceFiles.add(tsFileResource);
- sourceFileNames.add(tsFileResource.getTsFile().getName());
- }
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
TsFileResource targetResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
CompactionLogger compactionLogger = new CompactionLogger(new File(logFilePath));
@@ -813,4 +976,73 @@ public class SizeTieredCompactionRecoverTest {
FileUtils.deleteDirectory(new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
}
}
+
+ @Test
+ public void testInnerRecoverWithIncompleteTargetFileUsingFilePathAndChangingDataDirs()
+ throws Exception {
+ try {
+ List<TsFileResource> sourceFiles = getSourceFiles();
+ List<String> sourceFileNames = new ArrayList<>();
+ sourceFiles.forEach(f -> sourceFileNames.add(f.getTsFile().getName()));
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(sourceFiles, true);
+ SimpleCompactionLogger compactionLogger = new SimpleCompactionLogger(new File(logFilePath));
+ compactionLogger.logSourceFiles(sourceFiles);
+ compactionLogger.logTargetFile(targetResource);
+ compactionLogger.close();
+ performer.setSourceFiles(sourceFiles);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new FastCompactionTaskSummary());
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true);
+ FileChannel channel = targetStream.getChannel();
+ channel.truncate(targetResource.getTsFile().length() - 100);
+ channel.close();
+ FileUtils.moveDirectory(
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"),
+ new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ setDataDirs(new String[][] {{TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"}});
+ InnerSpaceCompactionTask recoverTask =
+ new InnerSpaceCompactionTask(
+ COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath));
+ recoverTask.recover();
+ // all the source file should be deleted
+ for (String sourceFileName : sourceFileNames) {
+ Assert.assertTrue(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ sourceFileName)
+ .exists());
+ }
+ Assert.assertFalse(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ + File.separator
+ + "data1"
+ + File.separator
+ + "sequence"
+ + File.separator
+ + COMPACTION_TEST_SG
+ + File.separator
+ + "0"
+ + File.separator
+ + "0",
+ targetResource.getTsFile().getName())
+ .exists());
+ } finally {
+ FileUtils.deleteDirectory(new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data1"));
+ }
+ }
}