You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2023/06/23 17:32:26 UTC
[iotdb] branch rel/1.1 updated: [IOTDB-6015][IOTDB-6023] load TsFile bugs: Not checking whether the tsfile data loaded locally is in the same time partition during the loading process & LoadTsFilePieceNode error when loading tsfile with empty value chunks (#10284)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new dafd53c8a85 [IOTDB-6015][IOTDB-6023] load TsFile bugs: Not checking whether the tsfile data loaded locally is in the same time partition during the loading process & LoadTsFilePieceNode error when loading tsfile with empty value chunks (#10284)
dafd53c8a85 is described below
commit dafd53c8a85246f8dff9e5c5536bf5eb051a9427
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Sat Jun 24 01:32:19 2023 +0800
[IOTDB-6015][IOTDB-6023] load TsFile bugs: Not checking whether the tsfile data loaded locally is in the same time partition during the loading process & LoadTsFilePieceNode error when loading tsfile with empty value chunks (#10284)
* [IOTDB-6015] Pipe: Not checking whether the tsfile data loaded locally is in the same time partition during the loading process (#10249)
When determining whether a tsfile should be loaded locally without splitting, you should also check that all the data in this tsfile is in the same dataregion and the same time division, without previously checking the latter
(cherry picked from commit 56faa8ef61b2cae499d1ef42fe994a8c4c9bc9c2)
* [IOTDB-6023] Pipe: LoadTsFilePieceNode error when loading tsfile with empty value chunks (#10281)
(cherry picked from commit 11755a10f5f237ba5dbdc4c4562222117e06dc9c)
---------
Co-authored-by: yschengzi <87...@users.noreply.github.com>
---
.../java/org/apache/iotdb/db/engine/load/TsFileSplitter.java | 8 ++++++--
.../plan/planner/plan/node/load/LoadSingleTsFileNode.java | 12 +++++++++++-
2 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java b/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java
index e3010f0eeb2..919c372c631 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java
@@ -229,7 +229,7 @@ public class TsFileSplitter {
chunkMetadata = offset2ChunkMetadata.get(chunkOffset - Byte.BYTES);
header = reader.readChunkHeader(marker);
if (header.getDataSize() == 0) {
- handleEmptyValueChunk(header, pageIndex2ChunkData);
+ handleEmptyValueChunk(header, pageIndex2ChunkData, chunkMetadata);
break;
}
@@ -421,12 +421,16 @@ public class TsFileSplitter {
}
private void handleEmptyValueChunk(
- ChunkHeader header, Map<Integer, List<AlignedChunkData>> pageIndex2ChunkData) {
+ ChunkHeader header,
+ Map<Integer, List<AlignedChunkData>> pageIndex2ChunkData,
+ IChunkMetadata chunkMetadata)
+ throws IOException {
Set<ChunkData> allChunkData = new HashSet<>();
for (Map.Entry<Integer, List<AlignedChunkData>> entry : pageIndex2ChunkData.entrySet()) {
for (AlignedChunkData alignedChunkData : entry.getValue()) {
if (!allChunkData.contains(alignedChunkData)) {
alignedChunkData.addValueChunk(header);
+ alignedChunkData.writeEntireChunk(ByteBuffer.allocate(0), chunkMetadata);
allChunkData.add(alignedChunkData);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
index d68184b04d3..18627c7be19 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
@@ -86,7 +86,17 @@ public class LoadSingleTsFileNode extends WritePlanNode {
slotList.add(
new Pair<>(o, TimePartitionUtils.getTimePartition(resource.getEndTime(o))));
});
- needDecodeTsFile = !isDispatchedToLocal(new HashSet<>(partitionFetcher.apply(slotList)));
+
+ if (slotList.isEmpty()) {
+ throw new IllegalStateException(
+ String.format("Devices in TsFile %s is empty, this should not happen here.", tsFile));
+ } else if (slotList.stream()
+ .anyMatch(slotPair -> !slotPair.getRight().equals(slotList.get(0).right))) {
+ needDecodeTsFile = true;
+ } else {
+ needDecodeTsFile = !isDispatchedToLocal(new HashSet<>(partitionFetcher.apply(slotList)));
+ }
+
if (!needDecodeTsFile && !resource.resourceFileExists()) {
resource.serialize();
}