You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/01/18 07:41:43 UTC
[iotdb] branch AlignedReaderSelfCheck created (now 0c76cea)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch AlignedReaderSelfCheck
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 0c76cea change self check
This branch includes the following new commits:
new fc0b916 change self check
new 0c76cea change self check
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: change self check
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch AlignedReaderSelfCheck
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0c76ceaf7276270b1b068a5f29b5a77cb103d509
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Jan 18 15:41:00 2022 +0800
change self check
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 112 +++++++++++----------
1 file changed, 57 insertions(+), 55 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index c504174..5e307a8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1274,63 +1274,65 @@ public class TsFileSequenceReader implements AutoCloseable {
Statistics<? extends Serializable> chunkStatistics =
Statistics.getStatsByType(dataType);
int dataSize = chunkHeader.getDataSize();
- if (((byte) (chunkHeader.getChunkType() & 0x3F)) == MetaMarker.CHUNK_HEADER) {
- while (dataSize > 0) {
- // a new Page
- PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), true);
- chunkStatistics.mergeStatistics(pageHeader.getStatistics());
- this.skipPageData(pageHeader);
- dataSize -= pageHeader.getSerializedPageSize();
- chunkHeader.increasePageNums(1);
- }
- } else {
- // only one page without statistic, we need to iterate each point to generate
- // statistic
- PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), false);
- Decoder valueDecoder =
- Decoder.getDecoderByType(
- chunkHeader.getEncodingType(), chunkHeader.getDataType());
- ByteBuffer pageData = readPage(pageHeader, chunkHeader.getCompressionType());
- Decoder timeDecoder =
- Decoder.getDecoderByType(
- TSEncoding.valueOf(
- TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
- TSDataType.INT64);
- PageReader reader =
- new PageReader(
- pageHeader,
- pageData,
- chunkHeader.getDataType(),
- valueDecoder,
- timeDecoder,
- null);
- BatchData batchData = reader.getAllSatisfiedPageData();
- while (batchData.hasCurrent()) {
- switch (dataType) {
- case INT32:
- chunkStatistics.update(batchData.currentTime(), batchData.getInt());
- break;
- case INT64:
- chunkStatistics.update(batchData.currentTime(), batchData.getLong());
- break;
- case FLOAT:
- chunkStatistics.update(batchData.currentTime(), batchData.getFloat());
- break;
- case DOUBLE:
- chunkStatistics.update(batchData.currentTime(), batchData.getDouble());
- break;
- case BOOLEAN:
- chunkStatistics.update(batchData.currentTime(), batchData.getBoolean());
- break;
- case TEXT:
- chunkStatistics.update(batchData.currentTime(), batchData.getBinary());
- break;
- default:
- throw new IOException("Unexpected type " + dataType);
+ if (dataSize > 0) {
+ if (((byte) (chunkHeader.getChunkType() & 0x3F)) == MetaMarker.CHUNK_HEADER) {
+ while (dataSize > 0) {
+ // a new Page
+ PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), true);
+ chunkStatistics.mergeStatistics(pageHeader.getStatistics());
+ this.skipPageData(pageHeader);
+ dataSize -= pageHeader.getSerializedPageSize();
+ chunkHeader.increasePageNums(1);
}
- batchData.next();
+ } else {
+ // only one page without statistic, we need to iterate each point to generate
+ // statistic
+ PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), false);
+ Decoder valueDecoder =
+ Decoder.getDecoderByType(
+ chunkHeader.getEncodingType(), chunkHeader.getDataType());
+ ByteBuffer pageData = readPage(pageHeader, chunkHeader.getCompressionType());
+ Decoder timeDecoder =
+ Decoder.getDecoderByType(
+ TSEncoding.valueOf(
+ TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+ TSDataType.INT64);
+ PageReader reader =
+ new PageReader(
+ pageHeader,
+ pageData,
+ chunkHeader.getDataType(),
+ valueDecoder,
+ timeDecoder,
+ null);
+ BatchData batchData = reader.getAllSatisfiedPageData();
+ while (batchData.hasCurrent()) {
+ switch (dataType) {
+ case INT32:
+ chunkStatistics.update(batchData.currentTime(), batchData.getInt());
+ break;
+ case INT64:
+ chunkStatistics.update(batchData.currentTime(), batchData.getLong());
+ break;
+ case FLOAT:
+ chunkStatistics.update(batchData.currentTime(), batchData.getFloat());
+ break;
+ case DOUBLE:
+ chunkStatistics.update(batchData.currentTime(), batchData.getDouble());
+ break;
+ case BOOLEAN:
+ chunkStatistics.update(batchData.currentTime(), batchData.getBoolean());
+ break;
+ case TEXT:
+ chunkStatistics.update(batchData.currentTime(), batchData.getBinary());
+ break;
+ default:
+ throw new IOException("Unexpected type " + dataType);
+ }
+ batchData.next();
+ }
+ chunkHeader.increasePageNums(1);
}
- chunkHeader.increasePageNums(1);
}
currentChunk =
new ChunkMetadata(measurementID, dataType, fileOffsetOfChunk, chunkStatistics);
[iotdb] 01/02: change self check
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch AlignedReaderSelfCheck
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fc0b91689b6e5c85177b79e19daecb7a141f2dfc
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Jan 18 15:26:26 2022 +0800
change self check
---
.../java/org/apache/iotdb/tsfile/file/MetaMarker.java | 15 +++++++++++++++
.../apache/iotdb/tsfile/read/TsFileSequenceReader.java | 6 ++++--
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/MetaMarker.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/MetaMarker.java
index 2c45941..70d326d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/MetaMarker.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/MetaMarker.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.tsfile.file;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+
import java.io.IOException;
/** MetaMarker denotes the type of headers and footers. Enum is not used for space saving. */
@@ -44,6 +46,19 @@ public class MetaMarker {
/** Chunk header marker and this chunk has only one page. */
public static final byte ONLY_ONE_PAGE_CHUNK_HEADER = 5;
+ /** Time Chunk header marker and this chunk has more than one page. */
+ public static final byte TIME_CHUNK_HEADER = (byte) (CHUNK_HEADER | TsFileConstant.TIME_COLUMN_MASK);
+
+ /** Value Chunk header marker and this chunk has more than one page. */
+ public static final byte VALUE_CHUNK_HEADER = (byte) (CHUNK_HEADER | TsFileConstant.VALUE_COLUMN_MASK);
+
+ /** Time Chunk header marker and this chunk has only one page. */
+ public static final byte ONLY_ONE_PAGE_TIME_CHUNK_HEADER = (byte) (ONLY_ONE_PAGE_CHUNK_HEADER | TsFileConstant.TIME_COLUMN_MASK);
+
+ /** Value Chunk header marker and this chunk has only one page. */
+ public static final byte ONLY_ONE_PAGE_VALUE_CHUNK_HEADER = (byte) (ONLY_ONE_PAGE_CHUNK_HEADER | TsFileConstant.VALUE_COLUMN_MASK);
+
+
private MetaMarker() {}
public static void handleUnexpectedMarker(byte marker) throws IOException {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index bd42af8..c504174 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1252,9 +1252,11 @@ public class TsFileSequenceReader implements AutoCloseable {
while ((marker = this.readMarker()) != MetaMarker.SEPARATOR) {
switch (marker) {
case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.TIME_CHUNK_HEADER:
+ case MetaMarker.VALUE_CHUNK_HEADER:
case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
- case (byte) (MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER | 0x80):
- case (byte) (MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER | 0x40):
+ case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER:
fileOffsetOfChunk = this.position() - 1;
// if there is something wrong with a chunk, we will drop the whole ChunkGroup
// as different chunks may be created by the same insertions(sqls), and partial