You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/11/27 15:33:25 UTC
[iotdb] branch readData created (now f4574c8)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch readData
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at f4574c8 read data in batches
This branch includes the following new commits:
new f4574c8 read data in batches
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: read data in batches
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch readData
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f4574c84e339d6e598c721b2b74f52009a029406
Author: Alima777 <wx...@gmail.com>
AuthorDate: Sat Nov 27 23:32:37 2021 +0800
read data in batches
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 39 ++++++++++++++--------
1 file changed, 25 insertions(+), 14 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 6f5e088..bd82a72 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -67,6 +67,7 @@ public class TsFileSequenceReader implements AutoCloseable {
protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
private static final String METADATA_INDEX_NODE_DESERIALIZE_ERROR =
"Something error happened while deserializing MetadataIndexNode of file {}";
+ private static final int MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024;
protected String file;
protected TsFileInput tsFileInput;
protected long fileMetadataPos;
@@ -1000,23 +1001,33 @@ public class TsFileSequenceReader implements AutoCloseable {
*
* @param position the start position of data in the tsFileInput, or the current position if
* position = -1
- * @param size the size of data that want to read
+ * @param totalSize the size of data that want to read
* @return data that been read.
*/
- protected ByteBuffer readData(long position, int size) throws IOException {
- ByteBuffer buffer = ByteBuffer.allocate(size);
- if (position < 0) {
- if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) {
- throw new IOException("reach the end of the data");
+ protected ByteBuffer readData(long position, int totalSize) throws IOException {
+ int allocateSize = Math.min(MAX_READ_BUFFER_SIZE, totalSize);
+ int allocateNum = (int) Math.ceil((double) totalSize / allocateSize);
+ ByteBuffer buffer = ByteBuffer.allocate(totalSize);
+ int bufferLimit = 0;
+ for (int i = 0; i < allocateNum; i++) {
+ if (i == allocateNum - 1) {
+ allocateSize = totalSize - allocateSize * (allocateNum - 1);
}
- } else {
- long actualReadSize = ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size);
- if (actualReadSize != size) {
- throw new IOException(
- String.format(
- "reach the end of the data. Size of data that want to read: %s,"
- + "actual read size: %s, position: %s",
- size, actualReadSize, position));
+ bufferLimit += allocateSize;
+ buffer.limit(bufferLimit);
+ if (position < 0) {
+ if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != allocateSize) {
+ throw new IOException("reach the end of the data");
+ }
+ } else {
+ long actualReadSize = ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, allocateSize);
+ if (actualReadSize != allocateSize) {
+ throw new IOException(
+ String.format(
+ "reach the end of the data. Size of data that want to read: %s,"
+ + "actual read size: %s, position: %s",
+ allocateSize, actualReadSize, position));
+ }
}
}
buffer.flip();