You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/12/18 07:28:22 UTC
(iotdb) 04/04: fix load and query error
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch 2G_metadata_read
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e86c93b9c62a164eea83192be61bba82f7d48d76
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Dec 18 15:27:56 2023 +0800
fix load and query error
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 26 ++++++++++---------
...leSequenceReaderTimeseriesMetadataIterator.java | 30 +++++++++++++++++-----
2 files changed, 38 insertions(+), 18 deletions(-)
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index ee0d9c953d2..f7a032d3be0 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -518,18 +518,20 @@ public class TsFileSequenceReader implements AutoCloseable {
} catch (ByteBufferAllocateException e) {
// when the buffer length is over than Integer.MAX_VALUE,
// using tsFileInput to get timeseriesMetadataList
- tsFileInput.position(metadataIndexPair.left.getOffset());
- while (tsFileInput.position() < metadataIndexPair.right) {
- TimeseriesMetadata timeseriesMetadata;
- try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(tsFileInput, true);
- } catch (Exception e1) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e1;
- }
- if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
- timeseriesMetadataList.add(timeseriesMetadata);
+ synchronized (this) {
+ tsFileInput.position(metadataIndexPair.left.getOffset());
+ while (tsFileInput.position() < metadataIndexPair.right) {
+ TimeseriesMetadata timeseriesMetadata;
+ try {
+ timeseriesMetadata = TimeseriesMetadata.deserializeFrom(tsFileInput, true);
+ } catch (Exception e1) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e1;
+ }
+ if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
+ timeseriesMetadataList.add(timeseriesMetadata);
+ }
}
}
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index 7648e402e97..0b53757171c 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.read;
+import org.apache.iotdb.tsfile.exception.ByteBufferAllocateException;
import org.apache.iotdb.tsfile.exception.TsFileSequenceReaderTimeseriesMetadataIteratorException;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode;
@@ -151,12 +152,17 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
throw new TsFileSequenceReaderTimeseriesMetadataIteratorException(
"currentBuffer still has some data left before deserializeLeafMeasurement");
}
-
- currentBuffer = reader.readData(metadataIndexEntry.getOffset(), endOffset);
-
- timeseriesMetadataMap
- .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
- .addAll(deserializeTimeseriesMetadata());
+ try {
+ currentBuffer = reader.readData(metadataIndexEntry.getOffset(), endOffset);
+ timeseriesMetadataMap
+ .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
+ .addAll(deserializeTimeseriesMetadata());
+ } catch (ByteBufferAllocateException e) {
+ reader.position(metadataIndexEntry.getOffset());
+ timeseriesMetadataMap
+ .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
+ .addAll(deserializeTimeseriesMetadataUsingTsFileInput(endOffset));
+ }
}
private List<TimeseriesMetadata> deserializeTimeseriesMetadata() {
@@ -170,6 +176,18 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
return timeseriesMetadataList;
}
+ private List<TimeseriesMetadata> deserializeTimeseriesMetadataUsingTsFileInput(long endOffset)
+ throws IOException {
+ final List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ while (reader.position() < endOffset
+ && currentTimeseriesMetadataCount < MAX_TIMESERIES_METADATA_COUNT) {
+ timeseriesMetadataList.add(
+ TimeseriesMetadata.deserializeFrom(reader.tsFileInput, needChunkMetadata));
+ currentTimeseriesMetadataCount++;
+ }
+ return timeseriesMetadataList;
+ }
+
private void deserializeInternalNode(
MetadataIndexEntry metadataIndexEntry, long endOffset, boolean isLeafDevice)
throws IOException {