You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/06/09 08:36:49 UTC
[iotdb] branch add-java-doc updated: add java doc for ChunkHeader,
FileLoaderUtils, VectorChunkMetadata, VectorRimeSeriesMetadata,
VectorChunkReader, ValuePageReader, ValuePageReader, VectorPageReader
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch add-java-doc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/add-java-doc by this push:
new f963213 add java doc for ChunkHeader, FileLoaderUtils, VectorChunkMetadata, VectorRimeSeriesMetadata, VectorChunkReader, ValuePageReader, ValuePageReader, VectorPageReader
f963213 is described below
commit f963213698dba25561264555ef31f1405eb9136a
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Wed Jun 9 16:35:31 2021 +0800
add java doc for ChunkHeader, FileLoaderUtils, VectorChunkMetadata, VectorRimeSeriesMetadata, VectorChunkReader, ValuePageReader, ValuePageReader, VectorPageReader
---
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 19 +++++++++++++++++--
.../iotdb/tsfile/file/header/ChunkHeader.java | 3 +++
.../tsfile/file/metadata/VectorChunkMetadata.java | 2 ++
.../file/metadata/VectorTimeSeriesMetadata.java | 14 ++++++++++++++
.../tsfile/read/reader/chunk/VectorChunkReader.java | 20 ++++++++++++++++++--
.../tsfile/read/reader/page/ValuePageReader.java | 8 ++++++++
.../tsfile/read/reader/page/VectorChunkReader.java | 21 ---------------------
.../tsfile/read/reader/page/VectorPageReader.java | 7 +++++++
8 files changed, 69 insertions(+), 25 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 45fe675..1e8c51f 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -113,6 +113,7 @@ public class FileLoaderUtils {
// common path
ITimeSeriesMetadata timeSeriesMetadata;
+ // If the tsfile is closed, we need to load from tsfile
if (resource.isClosed()) {
if (!resource.getTsFile().exists()) {
return null;
@@ -130,7 +131,7 @@ public class FileLoaderUtils {
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
}
- } else {
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
timeSeriesMetadata = resource.getTimeSeriesMetadata();
if (timeSeriesMetadata != null) {
timeSeriesMetadata.setChunkMetadataLoader(
@@ -156,6 +157,14 @@ public class FileLoaderUtils {
return timeSeriesMetadata;
}
+ /**
+ * Load VectorTimeSeriesMetadata for Vector
+ *
+ * @param resource corresponding TsFileResource
+ * @param seriesPath instance of VectorPartialPath, vector's full path, e.g. (root.sg1.d1.vector,
+ * [root.sg1.d1.vector.s1, root.sg1.d1.vector.s2])
+ * @param subSensorList subSensorList of the seriesPath
+ */
private static VectorTimeSeriesMetadata loadVectorTimeSeriesMetadata(
TsFileResource resource,
PartialPath seriesPath,
@@ -165,10 +174,14 @@ public class FileLoaderUtils {
Set<String> allSensors)
throws IOException {
VectorTimeSeriesMetadata vectorTimeSeriesMetadata = null;
+ // If the tsfile is closed, we need to load from tsfile
if (resource.isClosed()) {
if (!resource.getTsFile().exists()) {
return null;
}
+ // load all the TimeseriesMetadata of vector, the first one is for time column and the
+ // remaining is for sub sensors
+ // the order of timeSeriesMetadata list is same as subSensorList's order
List<TimeseriesMetadata> timeSeriesMetadata =
TimeSeriesMetadataCache.getInstance()
.get(
@@ -181,6 +194,8 @@ public class FileLoaderUtils {
.collect(Collectors.toList()),
allSensors,
context.isDebug());
+
+ // assemble VectorTimeSeriesMetadata
if (timeSeriesMetadata != null && !timeSeriesMetadata.isEmpty()) {
timeSeriesMetadata
.get(0)
@@ -197,7 +212,7 @@ public class FileLoaderUtils {
timeSeriesMetadata.get(0),
timeSeriesMetadata.subList(1, timeSeriesMetadata.size()));
}
- } else {
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
vectorTimeSeriesMetadata = (VectorTimeSeriesMetadata) resource.getTimeSeriesMetadata();
if (vectorTimeSeriesMetadata != null) {
vectorTimeSeriesMetadata.setChunkMetadataLoader(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index 22da502..a79db95 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -38,6 +38,9 @@ public class ChunkHeader {
/**
* 1 means this chunk has more than one page, so each page has its own page statistic 5 means this
* chunk has only one page, and this page has no page statistic
+ *
+ * <p>if the 8th bit of this byte is 1 means this chunk is a time chunk of one vector if the 7th
+ * bit of this byte is 1 means this chunk is a value chunk of one vector
*/
private byte chunkType;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java
index 01f617a..6558da0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorChunkMetadata.java
@@ -31,7 +31,9 @@ import java.util.List;
public class VectorChunkMetadata implements IChunkMetadata {
+ // ChunkMetadata for time column
private final IChunkMetadata timeChunkMetadata;
+ // ChunkMetadata for all subSensors in the vector
private final List<IChunkMetadata> valueChunkMetadataList;
public VectorChunkMetadata(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java
index f82194d..87e88af 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/VectorTimeSeriesMetadata.java
@@ -27,7 +27,9 @@ import java.util.List;
public class VectorTimeSeriesMetadata implements ITimeSeriesMetadata {
+ // TimeSeriesMetadata for time column
private final TimeseriesMetadata timeseriesMetadata;
+ // TimeSeriesMetadata for all subSensors in the vector
private final List<TimeseriesMetadata> valueTimeseriesMetadataList;
public VectorTimeSeriesMetadata(
@@ -36,6 +38,10 @@ public class VectorTimeSeriesMetadata implements ITimeSeriesMetadata {
this.valueTimeseriesMetadataList = valueTimeseriesMetadataList;
}
+ /**
+ * If the vector contains only one sub sensor, just return the sub sensor's Statistics Otherwise,
+ * return the Statistics of the time column
+ */
@Override
public Statistics getStatistics() {
return valueTimeseriesMetadataList.size() == 1
@@ -69,6 +75,14 @@ public class VectorTimeSeriesMetadata implements ITimeSeriesMetadata {
}
}
+ /**
+ * If the chunkMetadataLoader is MemChunkMetadataLoader, the VectorChunkMetadata is already
+ * assembled while constructing the in-memory TsFileResource, so we just return the assembled
+ * VectorChunkMetadata list.
+ *
+ * <p>Otherwise, we need to assemble the ChunkMetadata of time column and the ChunkMetadata of all
+ * the subSensors to generate the VectorChunkMetadata
+ */
@Override
public List<IChunkMetadata> loadChunkMetadataList() throws IOException {
if (timeseriesMetadata.getChunkMetadataLoader().isMemChunkMetadataLoader()) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java
index 37afc31..3cb62c8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/VectorChunkReader.java
@@ -43,9 +43,13 @@ import java.util.List;
public class VectorChunkReader implements IChunkReader {
+ // chunk header of the time column
private final ChunkHeader timeChunkHeader;
+ // chunk headers of all the sub sensors
private final List<ChunkHeader> valueChunkHeaderList = new ArrayList<>();
+ // chunk data of the time column
private final ByteBuffer timeChunkDataBuffer;
+ // chunk data of all the sub sensors
private final List<ByteBuffer> valueChunkDataBufferList = new ArrayList<>();
private final IUnCompressor unCompressor;
private final Decoder timeDecoder =
@@ -83,6 +87,7 @@ public class VectorChunkReader implements IChunkReader {
initAllPageReaders(timeChunk.getChunkStatistic(), valueChunkStatisticsList);
}
+ /** construct all the page readers in this chunk */
private void initAllPageReaders(
Statistics timeChunkStatistics, List<Statistics> valueChunkStatisticsList)
throws IOException {
@@ -91,6 +96,8 @@ public class VectorChunkReader implements IChunkReader {
// deserialize a PageHeader from chunkDataBuffer
PageHeader timePageHeader;
List<PageHeader> valuePageHeaderList = new ArrayList<>();
+ // mask the two highest bit
+ // this chunk has only one page
if ((timeChunkHeader.getChunkType() & 0x3F) == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) {
timePageHeader = PageHeader.deserializeFrom(timeChunkDataBuffer, timeChunkStatistics);
for (int i = 0; i < valueChunkDataBufferList.size(); i++) {
@@ -98,7 +105,7 @@ public class VectorChunkReader implements IChunkReader {
PageHeader.deserializeFrom(
valueChunkDataBufferList.get(i), valueChunkStatisticsList.get(i)));
}
- } else {
+ } else { // this chunk has more than one page
timePageHeader =
PageHeader.deserializeFrom(timeChunkDataBuffer, timeChunkHeader.getDataType());
for (int i = 0; i < valueChunkDataBufferList.size(); i++) {
@@ -145,6 +152,7 @@ public class VectorChunkReader implements IChunkReader {
List<TSDataType> valueDataTypeList = new ArrayList<>();
List<Decoder> valueDecoderList = new ArrayList<>();
for (int i = 0; i < valuePageHeader.size(); i++) {
+ // if the page is satisfied, deserialize it
if (pageSatisfied(valuePageHeader.get(i), valueDeleteIntervalList.get(i))) {
getPageInfo(
valuePageHeader.get(i),
@@ -155,7 +163,7 @@ public class VectorChunkReader implements IChunkReader {
valuePageDataList.add(valuePageInfo.pageData);
valueDataTypeList.add(valuePageInfo.dataType);
valueDecoderList.add(valuePageInfo.decoder);
- } else {
+ } else { // if the page is not satisfied, just skip it
valueChunkDataBufferList
.get(i)
.position(
@@ -181,6 +189,14 @@ public class VectorChunkReader implements IChunkReader {
return vectorPageReader;
}
+ /**
+ * deserialize the page
+ *
+ * @param pageHeader PageHeader for current page
+ * @param chunkBuffer current chunk data buffer
+ * @param chunkHeader current chunk header
+ * @param pageInfo A struct to put the deserialized page into.
+ */
private void getPageInfo(
PageHeader pageHeader, ByteBuffer chunkBuffer, ChunkHeader chunkHeader, PageInfo pageInfo)
throws IOException {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
index dd5eea3..dd0973e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
@@ -79,6 +79,10 @@ public class ValuePageReader {
this.valueBuffer = pageData.slice();
}
+ /**
+ * return a BatchData with the corresponding timeBatch, the BatchData's dataType is same as this
+ * sub sensor
+ */
public BatchData nextBatch(long[] timeBatch, boolean ascending, Filter filter) {
BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false);
for (int i = 0; i < timeBatch.length; i++) {
@@ -130,6 +134,10 @@ public class ValuePageReader {
return pageData.flip();
}
+ /**
+ * return the value array of the corresponding time, if this sub sensor don't have a value in a
+ * time, just fill it with null
+ */
public TsPrimitiveType[] nextValueBatch(long[] timeBatch) {
TsPrimitiveType[] valueBatch = new TsPrimitiveType[size];
if (valueBuffer == null) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorChunkReader.java
deleted file mode 100644
index b4665fc..0000000
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorChunkReader.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.read.reader.page;
-
-public class VectorChunkReader {}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java
index eeb095d..09a2ab7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/VectorPageReader.java
@@ -71,9 +71,13 @@ public class VectorPageReader implements IPageReader {
@Override
public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
long[] timeBatch = timePageReader.nexTimeBatch();
+ // if the vector contains only one sub sensor, just return a common BatchData whose DataType is
+ // same as the only one sub sensor.
if (valuePageReaderList.size() == 1) {
return valuePageReaderList.get(0).nextBatch(timeBatch, ascending, filter);
}
+
+ // if the vector contains more than on sub sensor, the BatchData's DataType is Vector
List<TsPrimitiveType[]> valueBatchList = new ArrayList<>(valueCount);
for (ValuePageReader valuePageReader : valuePageReaderList) {
valueBatchList.add(valuePageReader.nextValueBatch(timeBatch));
@@ -81,6 +85,7 @@ public class VectorPageReader implements IPageReader {
BatchData pageData = BatchDataFactory.createBatchData(TSDataType.VECTOR, ascending, false);
boolean isNull;
for (int i = 0; i < timeBatch.length; i++) {
+ // used to record whether the sub sensors are all null in current time
isNull = true;
TsPrimitiveType[] v = new TsPrimitiveType[valueCount];
for (int j = 0; j < v.length; j++) {
@@ -89,6 +94,8 @@ public class VectorPageReader implements IPageReader {
isNull = false;
}
}
+ // if all the sub sensors' value are null in current time
+ // or current row is not satisfied with the filter, just discard it
// TODO fix value filter v[0].getValue()
if (!isNull && (filter == null || filter.satisfy(timeBatch[i], v[0].getValue()))) {
pageData.putVector(timeBatch[i], v);