You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/11/28 04:00:39 UTC
[iotdb] branch rel/0.12 updated: limit the single io size when using FileChannel (#4491)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 3dd0a04 limit the single io size when using FileChannel (#4491)
3dd0a04 is described below
commit 3dd0a04d8377848e6cb691357485bfcdee6e3f75
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Sat Nov 27 22:00:10 2021 -0600
limit the single io size when using FileChannel (#4491)
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 41 ++++++++++++++--------
1 file changed, 27 insertions(+), 14 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index d174ca0..40ebdf0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -83,6 +83,7 @@ public class TsFileSequenceReader implements AutoCloseable {
protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
private static final String METADATA_INDEX_NODE_DESERIALIZE_ERROR =
"Something error happened while deserializing MetadataIndexNode of file {}";
+ private static final int MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024;
protected String file;
protected TsFileInput tsFileInput;
protected long fileMetadataPos;
@@ -900,24 +901,36 @@ public class TsFileSequenceReader implements AutoCloseable {
*
* @param position the start position of data in the tsFileInput, or the current position if
* position = -1
- * @param size the size of data that want to read
+ * @param totalSize the size of data that want to read
* @return data that been read.
*/
- protected ByteBuffer readData(long position, int size) throws IOException {
- ByteBuffer buffer = ByteBuffer.allocate(size);
- if (position < 0) {
- if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) {
- throw new IOException("reach the end of the data");
+ protected ByteBuffer readData(long position, int totalSize) throws IOException {
+ int allocateSize = Math.min(MAX_READ_BUFFER_SIZE, totalSize);
+ int allocateNum = (int) Math.ceil((double) totalSize / allocateSize);
+ ByteBuffer buffer = ByteBuffer.allocate(totalSize);
+ int bufferLimit = 0;
+ for (int i = 0; i < allocateNum; i++) {
+ if (i == allocateNum - 1) {
+ allocateSize = totalSize - allocateSize * (allocateNum - 1);
}
- } else {
- long actualReadSize = ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size);
- if (actualReadSize != size) {
- throw new IOException(
- String.format(
- "reach the end of the data. Size of data that want to read: %s,"
- + "actual read size: %s, position: %s",
- size, actualReadSize, position));
+ bufferLimit += allocateSize;
+ buffer.limit(bufferLimit);
+ if (position < 0) {
+ if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != allocateSize) {
+ throw new IOException("reach the end of the data");
+ }
+ } else {
+ long actualReadSize =
+ ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, allocateSize);
+ if (actualReadSize != allocateSize) {
+ throw new IOException(
+ String.format(
+ "reach the end of the data. Size of data that want to read: %s,"
+ + "actual read size: %s, position: %s",
+ allocateSize, actualReadSize, position));
+ }
}
+ position += allocateSize;
}
buffer.flip();
return buffer;