You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/01/20 10:23:19 UTC
[iotdb] 01/02: some changes
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch NewTsFileV2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 140ddec0e738ee4ed85ebfb999444639f8d329ba
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Tue Jan 19 13:54:10 2021 +0800
some changes
---
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 7 ++---
.../tsfile/file/metadata/TimeseriesMetadata.java | 32 ++++++++++++++++++++--
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 8 +++---
4 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 384a496..a0cf793 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -193,7 +193,7 @@ public class TimeSeriesMetadataCache {
"Get timeseries: {}.{} metadata in file: {} from cache: {}.", key.device,
key.measurement, key.filePath, timeseriesMetadata);
}
- return new TimeseriesMetadata(timeseriesMetadata);
+ return timeseriesMetadata;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 7144419..2b55d3b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.db.query.reader.chunk.metadata;
import java.io.IOException;
import java.util.List;
-import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.metadata.PartialPath;
@@ -49,10 +48,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
}
@Override
- public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
- throws IOException {
- List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache
- .getInstance().get(resource.getTsFilePath(), seriesPath, timeseriesMetadata);
+ public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata) {
+ List<ChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList();
setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 4f93f17..a8c30dd 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -22,11 +22,13 @@ package org.apache.iotdb.tsfile.file.metadata;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.tsfile.common.cache.Accountable;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
+import org.apache.iotdb.tsfile.utils.PublicBAOS;
import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -56,18 +58,25 @@ public class TimeseriesMetadata implements Accountable {
// used for SeriesReader to indicate whether it is a seq/unseq timeseries metadata
private boolean isSeq = true;
+ private ByteBuffer chunkMetadataListByteBuffer;
+
+ private PublicBAOS chunkMetadataListBuffer;
+
+ private ArrayList<ChunkMetadata> chunkMetadataList;
+
public TimeseriesMetadata() {
}
public TimeseriesMetadata(byte timeSeriesMetadataType, long startOffsetOfChunkMetaDataList,
int chunkMetaDataListDataSize, String measurementId, TSDataType dataType,
- Statistics statistics) {
+ Statistics statistics, PublicBAOS chunkMetadataListBuffer) {
this.timeSeriesMetadataType = timeSeriesMetadataType;
this.startOffsetOfChunkMetaDataList = startOffsetOfChunkMetaDataList;
this.chunkMetaDataListDataSize = chunkMetaDataListDataSize;
this.measurementId = measurementId;
this.dataType = dataType;
this.statistics = statistics;
+ this.chunkMetadataListBuffer = chunkMetadataListBuffer;
}
public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) {
@@ -86,9 +95,13 @@ public class TimeseriesMetadata implements Accountable {
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer));
timeseriesMetaData.setTSDataType(ReadWriteIOUtils.readDataType(buffer));
timeseriesMetaData.setOffsetOfChunkMetaDataList(ReadWriteIOUtils.readLong(buffer));
+ int chunkMetaDataListDataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
timeseriesMetaData
- .setDataSizeOfChunkMetaDataList(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
+ .setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize);
timeseriesMetaData.setStatistics(Statistics.deserialize(buffer, timeseriesMetaData.dataType));
+ timeseriesMetaData.chunkMetadataListByteBuffer = buffer.slice();
+ timeseriesMetaData.chunkMetadataListByteBuffer.limit(chunkMetaDataListDataSize);
+ buffer.position(buffer.position() + chunkMetaDataListDataSize);
return timeseriesMetaData;
}
@@ -108,6 +121,8 @@ public class TimeseriesMetadata implements Accountable {
byteLen += ReadWriteForEncodingUtils
.writeUnsignedVarInt(chunkMetaDataListDataSize, outputStream);
byteLen += statistics.serialize(outputStream);
+ chunkMetadataListBuffer.writeTo(outputStream);
+ byteLen += chunkMetadataListBuffer.size();
return byteLen;
}
@@ -167,6 +182,19 @@ public class TimeseriesMetadata implements Accountable {
return chunkMetadataLoader.loadChunkMetadataList(this);
}
+ public List<ChunkMetadata> getChunkMetadataList() {
+ if (chunkMetadataList == null) {
+ chunkMetadataList = new ArrayList<>();
+ while (chunkMetadataListByteBuffer.hasRemaining()) {
+ chunkMetadataList.add(ChunkMetadata.deserializeFrom(chunkMetadataListByteBuffer, this));
+ }
+ // minimize the storage of an ArrayList instance.
+ chunkMetadataList.trimToSize();
+ chunkMetadataListByteBuffer = null;
+ }
+ return chunkMetadataList;
+ }
+
public boolean isModified() {
return modified;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index d3290a9..f7e7d72 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -282,8 +282,8 @@ public class TsFileIOWriter {
String device = path.getDevice();
// create TimeseriesMetaData
+ PublicBAOS publicBAOS = new PublicBAOS();
TSDataType dataType = entry.getValue().get(entry.getValue().size() - 1).getDataType();
- long offsetOfChunkMetadataList = out.getPosition();
Statistics seriesStatistics = Statistics.getStatsByType(dataType);
int chunkMetadataListLength = 0;
@@ -294,12 +294,12 @@ public class TsFileIOWriter {
continue;
}
chunkMetadataListLength += chunkMetadata
- .serializeTo(out.wrapAsStream(), serializeStatistic);
+ .serializeTo(publicBAOS, serializeStatistic);
seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
}
TimeseriesMetadata timeseriesMetadata = new TimeseriesMetadata(
- serializeStatistic ? (byte) 1 : (byte) 0, offsetOfChunkMetadataList,
- chunkMetadataListLength, path.getMeasurement(), dataType, seriesStatistics);
+ serializeStatistic ? (byte) 1 : (byte) 0, 0,
+ chunkMetadataListLength, path.getMeasurement(), dataType, seriesStatistics, publicBAOS);
deviceTimeseriesMetadataMap.computeIfAbsent(device, k -> new ArrayList<>())
.add(timeseriesMetadata);
}