You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/03/02 03:07:47 UTC
[iotdb] branch rel/0.12 updated: [To rel/0.12][IOTDB-2640]Fix cross compaction recover bug (#5151)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 7356468 [To rel/0.12][IOTDB-2640]Fix cross compaction recover bug (#5151)
7356468 is described below
commit 735646843f72f7f754296ccc629f567b86bf8b3e
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Wed Mar 2 11:06:02 2022 +0800
[To rel/0.12][IOTDB-2640]Fix cross compaction recover bug (#5151)
---
.../db/engine/merge/recover/MergeLogAnalyzer.java | 15 +++
.../iotdb/db/engine/merge/MergeRecoverTest.java | 117 ++++++++++++++++++++-
2 files changed, 128 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogAnalyzer.java
index 1203964..4b13c3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogAnalyzer.java
@@ -110,6 +110,11 @@ public class MergeLogAnalyzer {
}
}
if (!currentFileFound) {
+ mergeSeqFiles.add(
+ new TsFileResource(
+ new File(
+ resource.getSeqFiles().get(0).getTsFile().getParent(),
+ MergeFileInfo.getFileInfoFromString(currLine).filename)));
allSourceFileExists = false;
}
}
@@ -150,6 +155,16 @@ public class MergeLogAnalyzer {
}
}
if (!currentFileFound) {
+ mergeUnseqFiles.add(
+ new TsFileResource(
+ new File(
+ resource
+ .getSeqFiles()
+ .get(0)
+ .getTsFile()
+ .getParent()
+ .replace("sequence", "unsequence"),
+ MergeFileInfo.getFileInfoFromString(currLine).filename)));
allSourceFileExists = false;
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeRecoverTest.java
index 21fac51..d237fcd 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeRecoverTest.java
@@ -106,6 +106,7 @@ public class MergeRecoverTest extends MergeTest {
@Test
public void testRecoverWithSomeSourceFilesLost() throws IOException, MetadataException {
sourceSeqFiles.get(0).getTsFile().delete();
+ tsFileManagement.remove(sourceSeqFiles.get(0), true);
RecoverMergeTask recoverMergeTask =
new RecoverMergeTask(
tsFileManagement,
@@ -137,6 +138,13 @@ public class MergeRecoverTest extends MergeTest {
}
Assert.assertFalse(mergingModsFile.exists());
Assert.assertFalse(logFile.exists());
+ Assert.assertEquals(10, tsFileManagement.getTsFileList(true).size());
+ Assert.assertEquals(0, tsFileManagement.getTsFileList(false).size());
+ for (int i = 0; i < tsFileManagement.getTsFileList(true).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(true).get(i).getTsFilePath(),
+ modifyTsFileNameUnseqMergCnt(tmpSourceSeqFiles.get(i).getTsFile()).getPath());
+ }
}
/**
@@ -154,6 +162,8 @@ public class MergeRecoverTest extends MergeTest {
.moveFile(
new File(sourceSeqFiles.get(0).getTsFilePath() + TsFileResource.RESOURCE_SUFFIX),
new File(targetFile.getPath() + TsFileResource.RESOURCE_SUFFIX));
+ tsFileManagement.remove(sourceSeqFiles.get(0), true);
+ tsFileManagement.add(new TsFileResource(targetFile), true);
targetFile = modifyTsFileNameUnseqMergCnt(sourceSeqFiles.get(1).getTsFile());
FSFactoryProducer.getFSFactory().moveFile(sourceSeqFiles.get(1).getTsFile(), targetFile);
// sg recover will serialize resouce file
@@ -162,6 +172,8 @@ public class MergeRecoverTest extends MergeTest {
FileLoaderUtils.updateTsFileResource(reader, targetTsFileResouce);
}
targetTsFileResouce.serialize();
+ tsFileManagement.remove(sourceSeqFiles.get(1), true);
+ tsFileManagement.add(targetTsFileResouce, true);
RecoverMergeTask recoverMergeTask =
new RecoverMergeTask(
@@ -194,6 +206,13 @@ public class MergeRecoverTest extends MergeTest {
}
Assert.assertFalse(mergingModsFile.exists());
Assert.assertFalse(logFile.exists());
+ Assert.assertEquals(10, tsFileManagement.getTsFileList(true).size());
+ Assert.assertEquals(0, tsFileManagement.getTsFileList(false).size());
+ for (int i = 0; i < tsFileManagement.getTsFileList(true).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(true).get(i).getTsFilePath(),
+ modifyTsFileNameUnseqMergCnt(tmpSourceSeqFiles.get(i).getTsFile()).getPath());
+ }
}
@Test
@@ -207,7 +226,10 @@ public class MergeRecoverTest extends MergeTest {
new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX),
new File(targetFile.getPath() + TsFileResource.RESOURCE_SUFFIX));
resource.remove();
+ tsFileManagement.remove(resource, true);
+ tsFileManagement.add(new TsFileResource(targetFile), true);
}
+
RecoverMergeTask recoverMergeTask =
new RecoverMergeTask(
tsFileManagement,
@@ -239,6 +261,73 @@ public class MergeRecoverTest extends MergeTest {
}
Assert.assertFalse(mergingModsFile.exists());
Assert.assertFalse(logFile.exists());
+ Assert.assertEquals(10, tsFileManagement.getTsFileList(true).size());
+ Assert.assertEquals(0, tsFileManagement.getTsFileList(false).size());
+ for (int i = 0; i < tsFileManagement.getTsFileList(true).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(true).get(i).getTsFilePath(),
+ modifyTsFileNameUnseqMergCnt(tmpSourceSeqFiles.get(i).getTsFile()).getPath());
+ }
+ }
+
+ @Test
+ public void testRecoverWithAllSourceSeqFilesAndSomeUnseqFilesLost()
+ throws IOException, MetadataException {
+ for (TsFileResource resource : sourceSeqFiles) {
+ File targetFile = modifyTsFileNameUnseqMergCnt(resource.getTsFile());
+ FSFactoryProducer.getFSFactory()
+ .moveFile(new File(resource.getTsFilePath() + MergeTask.MERGE_SUFFIX), targetFile);
+ FSFactoryProducer.getFSFactory()
+ .moveFile(
+ new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX),
+ new File(targetFile.getPath() + TsFileResource.RESOURCE_SUFFIX));
+ resource.remove();
+ tsFileManagement.remove(resource, true);
+ tsFileManagement.add(new TsFileResource(targetFile), true);
+ }
+ for (int i = 0; i < 3; i++) {
+ TsFileResource resource = sourceUnseqFiles.get(i);
+ resource.remove();
+ tsFileManagement.remove(resource, false);
+ }
+ RecoverMergeTask recoverMergeTask =
+ new RecoverMergeTask(
+ tsFileManagement,
+ TestConstant.SEQUENCE_DATA_DIR,
+ tsFileManagement::mergeEndAction,
+ "recoverTest",
+ true,
+ "root.sg1");
+ recoverMergeTask.recoverMerge();
+ for (TsFileResource seqResource : tmpSourceSeqFiles) {
+ Assert.assertFalse(seqResource.getTsFile().exists());
+ Assert.assertFalse(
+ new File(seqResource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists());
+ Assert.assertFalse(new File(seqResource.getTsFilePath() + MergeTask.MERGE_SUFFIX).exists());
+ File targetFile = modifyTsFileNameUnseqMergCnt(seqResource.getTsFile());
+ Assert.assertTrue(targetFile.exists());
+ Assert.assertTrue(new File(targetFile.getPath() + TsFileResource.RESOURCE_SUFFIX).exists());
+ Assert.assertFalse(seqResource.getModFile().exists());
+ ModificationFile targetModsFile =
+ new ModificationFile(targetFile.getPath() + ModificationFile.FILE_SUFFIX);
+ Assert.assertTrue(targetModsFile.exists());
+ Assert.assertEquals(2, targetModsFile.getModifications().size());
+ }
+ for (TsFileResource unseqResource : tmpSourceUnseqFiles) {
+ Assert.assertFalse(unseqResource.getTsFile().exists());
+ Assert.assertFalse(
+ new File(unseqResource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists());
+ Assert.assertFalse(unseqResource.getModFile().exists());
+ }
+ Assert.assertFalse(mergingModsFile.exists());
+ Assert.assertFalse(logFile.exists());
+ Assert.assertEquals(10, tsFileManagement.getTsFileList(true).size());
+ Assert.assertEquals(0, tsFileManagement.getTsFileList(false).size());
+ for (int i = 0; i < tsFileManagement.getTsFileList(true).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(true).get(i).getTsFilePath(),
+ modifyTsFileNameUnseqMergCnt(tmpSourceSeqFiles.get(i).getTsFile()).getPath());
+ }
}
@Test
@@ -252,9 +341,12 @@ public class MergeRecoverTest extends MergeTest {
new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX),
new File(targetFile.getPath() + TsFileResource.RESOURCE_SUFFIX));
resource.remove();
+ tsFileManagement.remove(resource, true);
+ tsFileManagement.add(new TsFileResource(targetFile), true);
}
for (TsFileResource resource : sourceUnseqFiles) {
resource.remove();
+ tsFileManagement.remove(resource, false);
}
RecoverMergeTask recoverMergeTask =
new RecoverMergeTask(
@@ -287,6 +379,13 @@ public class MergeRecoverTest extends MergeTest {
}
Assert.assertFalse(mergingModsFile.exists());
Assert.assertFalse(logFile.exists());
+ Assert.assertEquals(10, tsFileManagement.getTsFileList(true).size());
+ Assert.assertEquals(0, tsFileManagement.getTsFileList(false).size());
+ for (int i = 0; i < tsFileManagement.getTsFileList(true).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(true).get(i).getTsFilePath(),
+ modifyTsFileNameUnseqMergCnt(tmpSourceSeqFiles.get(i).getTsFile()).getPath());
+ }
}
@Test
@@ -321,6 +420,18 @@ public class MergeRecoverTest extends MergeTest {
}
Assert.assertFalse(mergingModsFile.exists());
Assert.assertFalse(logFile.exists());
+ Assert.assertEquals(10, tsFileManagement.getTsFileList(true).size());
+ Assert.assertEquals(5, tsFileManagement.getTsFileList(false).size());
+ for (int i = 0; i < tsFileManagement.getTsFileList(true).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(true).get(i).getTsFilePath(),
+ tmpSourceSeqFiles.get(i).getTsFilePath());
+ }
+ for (int i = 0; i < tsFileManagement.getTsFileList(false).size(); i++) {
+ Assert.assertEquals(
+ tsFileManagement.getTsFileList(false).get(i).getTsFilePath(),
+ tmpSourceUnseqFiles.get(i).getTsFilePath());
+ }
}
private void createFiles() throws IOException, IllegalPathException, WriteProcessException {
@@ -339,6 +450,7 @@ public class MergeRecoverTest extends MergeTest {
+ ".tsfile"));
Assert.assertTrue(file.createNewFile());
TsFileResource tsFileResource = new TsFileResource(file);
+ prepareFile(tsFileResource, i * 10, 10, i * 10);
tsFileResource.setClosed(true);
tsFileResource.setMinPlanIndex(i);
tsFileResource.setMaxPlanIndex(i);
@@ -352,8 +464,6 @@ public class MergeRecoverTest extends MergeTest {
deletion = new Deletion(new PartialPath("root.sg1.d1", "s0"), 1, 200, 300);
tsFileResource.getModFile().write(deletion);
tsFileResource.getModFile().close();
-
- prepareFile(tsFileResource, i * 10, 10, i * 10);
}
// create source unseq files
@@ -371,6 +481,7 @@ public class MergeRecoverTest extends MergeTest {
+ ".tsfile"));
Assert.assertTrue(file.createNewFile());
TsFileResource tsFileResource = new TsFileResource(file);
+ prepareFile(tsFileResource, i * 10, 5, i * 10);
tsFileResource.setClosed(true);
tsFileResource.setMinPlanIndex(i);
tsFileResource.setMaxPlanIndex(i);
@@ -384,8 +495,6 @@ public class MergeRecoverTest extends MergeTest {
deletion = new Deletion(new PartialPath("root.sg1.d1", "s0"), 1, 200, 300);
tsFileResource.getModFile().write(deletion);
tsFileResource.getModFile().close();
-
- prepareFile(tsFileResource, i * 10, 5, i * 10);
}
// create .merge files