You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/01/20 10:37:39 UTC
[iotdb] branch NewTsFileV2 updated: change the sketch tool and
upgrade tool
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch NewTsFileV2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/NewTsFileV2 by this push:
new 9d8840d change the sketch tool and upgrade tool
9d8840d is described below
commit 9d8840d1dfdfca693f3b948d3ddbfccd5a9b6fcd
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Jan 20 18:37:16 2021 +0800
change the sketch tool and upgrade tool
---
.../apache/iotdb/db/tools/TsFileSketchTool.java | 8 +++----
.../tsfile/file/metadata/TimeseriesMetadata.java | 26 +++++++++++++---------
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 4 ++--
3 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index 8423ce0..4cc6552 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -115,19 +115,19 @@ public class TsFileSketchTool {
}
Map<String, List<TimeseriesMetadata>> allTimeseriesMetadata = reader.getAllTimeseriesMetadata();
- Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
+ Map<String, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
for (Map.Entry<String, List<TimeseriesMetadata>> entry : allTimeseriesMetadata.entrySet()) {
String device = entry.getKey();
List<TimeseriesMetadata> seriesMetadataList = entry.getValue();
for (TimeseriesMetadata seriesMetadata : seriesMetadataList) {
- timeseriesMetadataMap.put(seriesMetadata.getOffsetOfChunkMetaDataList(),
+ timeseriesMetadataMap.put(seriesMetadata.getMeasurementId(),
new Pair<>(new Path(device, seriesMetadata.getMeasurementId()), seriesMetadata));
}
}
- for (Map.Entry<Long, Pair<Path, TimeseriesMetadata>> entry : timeseriesMetadataMap.entrySet()) {
+ for (Map.Entry<String, Pair<Path, TimeseriesMetadata>> entry : timeseriesMetadataMap.entrySet()) {
printlnBoth(
- pw, String.format("%20s", entry.getKey())
+ pw, entry.getKey()
+ "|\t[ChunkMetadataList] of " + entry.getValue().left
+ ", tsDataType:" + entry.getValue().right.getTSDataType());
printlnBoth(pw,
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 13ff2a0..e961a3a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -35,12 +35,19 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class TimeseriesMetadata implements Accountable {
/**
- * 0 means this time series has only one chunk, no need to save the statistic again in chunk metadata
- * 1 means this time series has more than one chunk, should save the statistic again in chunk metadata
+ * used for old version tsfile
+ */
+ @Deprecated
+ private long startOffsetOfChunkMetaDataList;
+
+
+ /**
+ * 0 means this time series has only one chunk, no need to save the statistic again in chunk
+ * metadata 1 means this time series has more than one chunk, should save the statistic again in
+ * chunk metadata
*/
private byte timeSeriesMetadataType;
- private long startOffsetOfChunkMetaDataList;
private int chunkMetaDataListDataSize;
private String measurementId;
@@ -66,11 +73,10 @@ public class TimeseriesMetadata implements Accountable {
public TimeseriesMetadata() {
}
- public TimeseriesMetadata(byte timeSeriesMetadataType, long startOffsetOfChunkMetaDataList,
- int chunkMetaDataListDataSize, String measurementId, TSDataType dataType,
- Statistics statistics, PublicBAOS chunkMetadataListBuffer) {
+ public TimeseriesMetadata(byte timeSeriesMetadataType, int chunkMetaDataListDataSize,
+ String measurementId, TSDataType dataType, Statistics statistics,
+ PublicBAOS chunkMetadataListBuffer) {
this.timeSeriesMetadataType = timeSeriesMetadataType;
- this.startOffsetOfChunkMetaDataList = startOffsetOfChunkMetaDataList;
this.chunkMetaDataListDataSize = chunkMetaDataListDataSize;
this.measurementId = measurementId;
this.dataType = dataType;
@@ -80,7 +86,6 @@ public class TimeseriesMetadata implements Accountable {
public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) {
this.timeSeriesMetadataType = timeseriesMetadata.timeSeriesMetadataType;
- this.startOffsetOfChunkMetaDataList = timeseriesMetadata.startOffsetOfChunkMetaDataList;
this.chunkMetaDataListDataSize = timeseriesMetadata.chunkMetaDataListDataSize;
this.measurementId = timeseriesMetadata.measurementId;
this.dataType = timeseriesMetadata.dataType;
@@ -94,7 +99,6 @@ public class TimeseriesMetadata implements Accountable {
timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(buffer));
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer));
timeseriesMetaData.setTSDataType(ReadWriteIOUtils.readDataType(buffer));
- timeseriesMetaData.setOffsetOfChunkMetaDataList(ReadWriteIOUtils.readLong(buffer));
int chunkMetaDataListDataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
timeseriesMetaData
.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize);
@@ -104,7 +108,8 @@ public class TimeseriesMetadata implements Accountable {
byteBuffer.limit(chunkMetaDataListDataSize);
timeseriesMetaData.chunkMetadataList = new ArrayList<>();
while (byteBuffer.hasRemaining()) {
- timeseriesMetaData.chunkMetadataList.add(ChunkMetadata.deserializeFrom(byteBuffer, timeseriesMetaData));
+ timeseriesMetaData.chunkMetadataList
+ .add(ChunkMetadata.deserializeFrom(byteBuffer, timeseriesMetaData));
}
// minimize the storage of an ArrayList instance.
timeseriesMetaData.chunkMetadataList.trimToSize();
@@ -125,7 +130,6 @@ public class TimeseriesMetadata implements Accountable {
byteLen += ReadWriteIOUtils.write(timeSeriesMetadataType, outputStream);
byteLen += ReadWriteIOUtils.writeVar(measurementId, outputStream);
byteLen += ReadWriteIOUtils.write(dataType, outputStream);
- byteLen += ReadWriteIOUtils.write(startOffsetOfChunkMetaDataList, outputStream);
byteLen += ReadWriteForEncodingUtils
.writeUnsignedVarInt(chunkMetaDataListDataSize, outputStream);
byteLen += statistics.serialize(outputStream);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index f7e7d72..18952eb 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -298,8 +298,8 @@ public class TsFileIOWriter {
seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
}
TimeseriesMetadata timeseriesMetadata = new TimeseriesMetadata(
- serializeStatistic ? (byte) 1 : (byte) 0, 0,
- chunkMetadataListLength, path.getMeasurement(), dataType, seriesStatistics, publicBAOS);
+ serializeStatistic ? (byte) 1 : (byte) 0, chunkMetadataListLength, path.getMeasurement(),
+ dataType, seriesStatistics, publicBAOS);
deviceTimeseriesMetadataMap.computeIfAbsent(device, k -> new ArrayList<>())
.add(timeseriesMetadata);
}