You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/05/24 11:39:44 UTC
[iotdb] branch rel/0.11 updated: fix unseq compaction loss data bug
(#3254)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new 94ef723 fix unseq compaction loss data bug (#3254)
94ef723 is described below
commit 94ef723f0b858eed72eaa67591af1c531eb09a71
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Mon May 24 19:39:07 2021 +0800
fix unseq compaction loss data bug (#3254)
fix unseq compaction loss data bug
---
.../iotdb/db/engine/merge/task/MergeMultiChunkTask.java | 13 +++++++++++--
.../org/apache/iotdb/db/engine/merge/MergeTaskTest.java | 15 ++++++++++-----
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java
index c5fd3fd..25f17b3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java
@@ -194,6 +194,16 @@ public class MergeMultiChunkTask {
mergeLogger.logTSEnd();
}
+ private String getMaxSensor(List<PartialPath> sensors) {
+ String maxSensor = sensors.get(0).getMeasurement();
+ for (int i = 1; i < sensors.size(); i++) {
+ if (maxSensor.compareTo(sensors.get(i).getMeasurement()) < 0) {
+ maxSensor = sensors.get(i).getMeasurement();
+ }
+ }
+ return maxSensor;
+ }
+
private void pathsMergeOneFile(int seqFileIdx, IPointReader[] unseqReaders) throws IOException {
TsFileResource currTsFile = resource.getSeqFiles().get(seqFileIdx);
String deviceId = currMergingPaths.get(0).getDevice();
@@ -237,7 +247,7 @@ public class MergeMultiChunkTask {
return;
}
- String lastSensor = currMergingPaths.get(currMergingPaths.size() - 1).getMeasurement();
+ String lastSensor = getMaxSensor(currMergingPaths);
String currSensor = null;
Map<String, List<ChunkMetadata>> measurementChunkMetadataListMap = new TreeMap<>();
// find all sensor to merge in order, if exceed, then break
@@ -447,7 +457,6 @@ public class MergeMultiChunkTask {
IChunkWriter chunkWriter,
TsFileResource currFile)
throws IOException {
-
int unclosedChunkPoint = lastUnclosedChunkPoint;
boolean chunkModified =
(currMeta.getDeleteIntervalList() != null && !currMeta.getDeleteIntervalList().isEmpty());
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
index ebbf8a9..dded161 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
@@ -105,23 +105,28 @@ public class MergeTaskTest extends MergeTest {
new MergeTask(new MergeResource(seqResources, unseqResources), tempSGDir.getPath(),
(k, v, l) -> {
}, "test",
- true, 1, MERGE_TEST_SG);
+ true, 10, MERGE_TEST_SG);
mergeTask.call();
QueryContext context = new QueryContext();
PartialPath path = new PartialPath(
- deviceIds[0] + TsFileConstant.PATH_SEPARATOR + measurementSchemas[0].getMeasurementId());
+ deviceIds[0] + TsFileConstant.PATH_SEPARATOR + measurementSchemas[9].getMeasurementId());
List<TsFileResource> list = new ArrayList<>();
list.add(seqResources.get(0));
- IBatchReader tsFilesReader = new SeriesRawDataBatchReader(path, measurementSchemas[0].getType(),
+ IBatchReader tsFilesReader = new SeriesRawDataBatchReader(path,
+ measurementSchemas[9].getType(),
context,
list, new ArrayList<>(), null, null, true);
+ long count = 0L;
while (tsFilesReader.hasNextBatch()) {
BatchData batchData = tsFilesReader.nextBatch();
- for (int i = 0; i < batchData.length(); i++) {
- assertEquals(batchData.getTimeByIndex(i) + 20000.0, batchData.getDoubleByIndex(i), 0.001);
+ for (int t = 0; t < batchData.length(); t++) {
+ assertEquals(batchData.getTimeByIndex(t) + 20000.0, batchData.getDoubleByIndex(t),
+ 0.001);
+ count++;
}
}
+ assertEquals(100,count);
tsFilesReader.close();
}