[iotdb] 01/01: limit the io size each read

[qi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch limit_read_io
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 901a6de01dbcc9981a3a1dd254007f271f902728
Author: qiaojialin <64...@qq.com>
AuthorDate: Sun Nov 28 01:32:01 2021 +0800

    limit the io size each read
---
 .../iotdb/tsfile/read/TsFileSequenceReader.java    | 41 ++++++++++++++--------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index d174ca0..40ebdf0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -83,6 +83,7 @@ public class TsFileSequenceReader implements AutoCloseable {
   protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
   private static final String METADATA_INDEX_NODE_DESERIALIZE_ERROR =
       "Something error happened while deserializing MetadataIndexNode of file {}";
+  private static final int MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024;
   protected String file;
   protected TsFileInput tsFileInput;
   protected long fileMetadataPos;
@@ -900,24 +901,36 @@ public class TsFileSequenceReader implements AutoCloseable {
    *
    * @param position the start position of data in the tsFileInput, or the current position if
    *     position = -1
-   * @param size the size of data that want to read
+   * @param totalSize the size of data that want to read
    * @return data that been read.
    */
-  protected ByteBuffer readData(long position, int size) throws IOException {
-    ByteBuffer buffer = ByteBuffer.allocate(size);
-    if (position < 0) {
-      if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) {
-        throw new IOException("reach the end of the data");
+  protected ByteBuffer readData(long position, int totalSize) throws IOException {
+    int allocateSize = Math.min(MAX_READ_BUFFER_SIZE, totalSize);
+    int allocateNum = (int) Math.ceil((double) totalSize / allocateSize);
+    ByteBuffer buffer = ByteBuffer.allocate(totalSize);
+    int bufferLimit = 0;
+    for (int i = 0; i < allocateNum; i++) {
+      if (i == allocateNum - 1) {
+        allocateSize = totalSize - allocateSize * (allocateNum - 1);
       }
-    } else {
-      long actualReadSize = ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size);
-      if (actualReadSize != size) {
-        throw new IOException(
-            String.format(
-                "reach the end of the data. Size of data that want to read: %s,"
-                    + "actual read size: %s, position: %s",
-                size, actualReadSize, position));
+      bufferLimit += allocateSize;
+      buffer.limit(bufferLimit);
+      if (position < 0) {
+        if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != allocateSize) {
+          throw new IOException("reach the end of the data");
+        }
+      } else {
+        long actualReadSize =
+            ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, allocateSize);
+        if (actualReadSize != allocateSize) {
+          throw new IOException(
+              String.format(
+                  "reach the end of the data. Size of data that want to read: %s,"
+                      + "actual read size: %s, position: %s",
+                  allocateSize, actualReadSize, position));
+        }
       }
+      position += allocateSize;
     }
     buffer.flip();
     return buffer;