You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/10/24 04:45:01 UTC
[incubator-iotdb] branch reduce_io_in_deserialize_chunkheader
created (now 45a72ad)
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a change to branch reduce_io_in_deserialize_chunkheader
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 45a72ad reduce IO in constructing chunk header
This branch includes the following new commits:
new 45a72ad reduce IO in constructing chunk header
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: reduce IO in constructing chunk header
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch reduce_io_in_deserialize_chunkheader
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 45a72ad5c9500500793c5bb9574a5b05bbdbc9d7
Author: suyue <23...@qq.com>
AuthorDate: Thu Oct 24 12:43:12 2019 +0800
reduce IO in constructing chunk header
---
.../apache/iotdb/tsfile/file/header/ChunkHeader.java | 17 +++++++++--------
.../apache/iotdb/tsfile/read/TsFileSequenceReader.java | 9 ++++++---
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
index da01cfc..b149ed3 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkHeader.java
@@ -76,7 +76,8 @@ public class ChunkHeader {
}
public static int getSerializedSize(String measurementID) {
- return Byte.BYTES + Integer.BYTES + getSerializedSize(measurementID.getBytes(TSFileConfig.STRING_CHARSET).length);
+ return Byte.BYTES + Integer.BYTES + getSerializedSize(
+ measurementID.getBytes(TSFileConfig.STRING_CHARSET).length);
}
private static int getSerializedSize(int measurementIdLength) {
@@ -135,29 +136,29 @@ public class ChunkHeader {
*
* @param input TsFileInput
* @param offset offset
+ * @param chunkHeaderSize the size of chunk's header
* @param markerRead read marker (boolean type)
* @return CHUNK_HEADER object
* @throws IOException IOException
*/
- public static ChunkHeader deserializeFrom(TsFileInput input, long offset, boolean markerRead)
+ public static ChunkHeader deserializeFrom(TsFileInput input, long offset, int chunkHeaderSize,
+ boolean markerRead)
throws IOException {
long offsetVar = offset;
if (!markerRead) {
offsetVar++;
}
- ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES + chunkHeaderSize);
input.read(buffer, offsetVar);
buffer.flip();
+
int size = buffer.getInt();
- offsetVar += Integer.BYTES;
- buffer = ByteBuffer.allocate(getSerializedSize(size));
- ReadWriteIOUtils.readAsPossible(input, offsetVar, buffer);
- buffer.flip();
String measurementID = ReadWriteIOUtils.readStringWithoutLength(buffer, size);
return deserializePartFrom(measurementID, buffer);
}
- private static ChunkHeader deserializePartFrom(String measurementID, ByteBuffer buffer) throws UnsupportedEncodingException {
+ private static ChunkHeader deserializePartFrom(String measurementID, ByteBuffer buffer)
+ throws UnsupportedEncodingException {
int dataSize = ReadWriteIOUtils.readInt(buffer);
TSDataType dataType = TSDataType.deserialize(ReadWriteIOUtils.readShort(buffer));
int numOfPages = ReadWriteIOUtils.readInt(buffer);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index f1a9cc3..b5f434b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -333,10 +333,12 @@ public class TsFileSequenceReader implements AutoCloseable {
* read the chunk's header.
*
* @param position the file offset of this chunk's header
+ * @param chunkHeaderSize the size of chunk's header
* @param markerRead true if the offset does not contains the marker , otherwise false
*/
- private ChunkHeader readChunkHeader(long position, boolean markerRead) throws IOException {
- return ChunkHeader.deserializeFrom(tsFileInput, position, markerRead);
+ private ChunkHeader readChunkHeader(long position, int chunkHeaderSize, boolean markerRead)
+ throws IOException {
+ return ChunkHeader.deserializeFrom(tsFileInput, position, chunkHeaderSize, markerRead);
}
/**
@@ -377,7 +379,8 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return -chunk
*/
public Chunk readMemChunk(ChunkMetaData metaData) throws IOException {
- ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader(), false);
+ int chunkHeadSize = ChunkHeader.getSerializedSize(metaData.getMeasurementUid());
+ ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader(), chunkHeadSize, false);
ByteBuffer buffer = readChunk(metaData.getOffsetOfChunkHeader() + header.getSerializedSize(),
header.getDataSize());
return new Chunk(header, buffer, metaData.getDeletedAt());