You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/03/15 13:51:15 UTC
[iotdb] branch rel/0.13 updated: [IOTDB-2723] Fix sequence inner space compaction loses data (#5249)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new b0d0af3 [IOTDB-2723] Fix sequence inner space compaction loses data (#5249)
b0d0af3 is described below
commit b0d0af35125856a0b22fd2df4b876cb540558274
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Tue Mar 15 21:49:48 2022 +0800
[IOTDB-2723] Fix sequence inner space compaction loses data (#5249)
---
.../utils/SingleSeriesCompactionExecutor.java | 9 ++-
.../InnerSpaceCompactionUtilsNoAlignedTest.java | 83 +++++++++++++++++++++-
2 files changed, 90 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
index 068ee2c..740d37c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
@@ -209,7 +209,14 @@ public class SingleSeriesCompactionExecutor {
}
private void writeCachedChunkIntoChunkWriter() throws IOException {
- cachedChunk.getData().flip();
+ if (cachedChunk.getData().position() != 0) {
+ // If the position of cache chunk data buffer is 0,
+ // it means that the cache chunk is the first chunk cached,
+ // and it hasn't merged with any chunk yet.
+ // If we flip it, both the position and limit in the buffer will be 0,
+ // which leads to the lost of data.
+ cachedChunk.getData().flip();
+ }
writeChunkIntoChunkWriter(cachedChunk);
cachedChunk = null;
cachedChunkMetadata = null;
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
index 78a8401..8251b0d 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
@@ -758,7 +758,7 @@ public class InnerSpaceCompactionUtilsNoAlignedTest {
}
@Test
- public void testMixCompact() throws Exception {
+ public void testMixCompact1() throws Exception {
long testTargetChunkPointNum = 2000L;
long testChunkSizeLowerBound = 1024L;
long testChunkPointNumLowerBound = 100L;
@@ -834,4 +834,85 @@ public class InnerSpaceCompactionUtilsNoAlignedTest {
.setChunkPointNumLowerBoundInCompaction(originChunkPointNumLowerBound);
}
}
+
+ @Test
+ public void testMixCompact2() throws Exception {
+ long testTargetChunkPointNum = 2000L;
+ long testChunkSizeLowerBound = 1024L;
+ long testChunkPointNumLowerBound = 100L;
+ long originTargetChunkSize = IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ long originTargetChunkPointNum =
+ IoTDBDescriptor.getInstance().getConfig().getTargetChunkPointNum();
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024 * 1024);
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(testTargetChunkPointNum);
+ long originChunkSizeLowerBound =
+ IoTDBDescriptor.getInstance().getConfig().getChunkSizeLowerBoundInCompaction();
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkSizeLowerBoundInCompaction(testChunkSizeLowerBound);
+ long originChunkPointNumLowerBound =
+ IoTDBDescriptor.getInstance().getConfig().getChunkPointNumLowerBoundInCompaction();
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkPointNumLowerBoundInCompaction(testChunkPointNumLowerBound);
+ try {
+ List<TsFileResource> sourceFiles = new ArrayList();
+ int fileNum = 12;
+ long pointStep = 10L;
+ long[] points = new long[] {1960, 50, 1960, 50, 2100, 50, 1960, 2300, 2500, 1000, 500, 500};
+ for (int i = 0; i < fileNum; ++i) {
+ List<List<Long>> chunkPagePointsNum = new ArrayList<>();
+ List<Long> pagePointsNum = new ArrayList<>();
+ pagePointsNum.add(points[i]);
+ chunkPagePointsNum.add(pagePointsNum);
+ TsFileResource resource =
+ new TsFileResource(new File(SEQ_DIRS, String.format("%d-%d-0-0.tsfile", i + 1, i + 1)));
+ sourceFiles.add(resource);
+ CompactionFileGeneratorUtils.writeTsFile(
+ fullPathSet, chunkPagePointsNum, i * 2500L, resource);
+ }
+
+ Map<PartialPath, List<TimeValuePair>> originData =
+ CompactionCheckerUtils.getDataByQuery(paths, schemaList, sourceFiles, new ArrayList<>());
+ TsFileNameGenerator.TsFileName tsFileName =
+ TsFileNameGenerator.getTsFileName(sourceFiles.get(0).getTsFile().getName());
+ TsFileResource targetResource =
+ new TsFileResource(
+ new File(
+ SEQ_DIRS,
+ String.format(
+ "%d-%d-%d-%d.tsfile",
+ tsFileName.getTime(),
+ tsFileName.getVersion(),
+ tsFileName.getInnerCompactionCnt() + 1,
+ tsFileName.getCrossCompactionCnt())));
+ InnerSpaceCompactionUtils.compact(targetResource, sourceFiles);
+ Map<String, List<List<Long>>> chunkPagePointsNumMerged = new HashMap<>();
+ // outer list is a chunk, inner list is point num in each page
+ for (String path : fullPathSet) {
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path, 2010);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path, 2010);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path, 2100);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path, 2010);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path, 2300);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path, 2500);
+ CompactionCheckerUtils.putChunk(
+ chunkPagePointsNumMerged, path, new long[] {1000, 500, 500});
+ }
+ Map<PartialPath, List<TimeValuePair>> compactedData =
+ CompactionCheckerUtils.getDataByQuery(
+ paths, schemaList, Collections.singletonList(targetResource), new ArrayList<>());
+ CompactionCheckerUtils.validDataByValueList(originData, compactedData);
+ CompactionCheckerUtils.checkChunkAndPage(chunkPagePointsNumMerged, targetResource);
+ } finally {
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(originTargetChunkSize);
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(originTargetChunkPointNum);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkSizeLowerBoundInCompaction(originChunkSizeLowerBound);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkPointNumLowerBoundInCompaction(originChunkPointNumLowerBound);
+ }
+ }
}