You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/12/23 15:53:56 UTC
[iotdb] branch master updated: Turn down the max capacity of tsblock from scanOperator (#8599)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 424528495c Turn down the max capacity of tsblock from scanOperator (#8599)
424528495c is described below
commit 424528495c5515c314f542d2523f7c0431ddc902
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Fri Dec 23 23:53:51 2022 +0800
Turn down the max capacity of tsblock from scanOperator (#8599)
---
.../src/assembly/resources/conf/iotdb-common.properties | 2 +-
.../org/apache/iotdb/db/mpp/execution/memory/MemoryPool.java | 12 ++++++++++++
.../iotdb/db/mpp/execution/exchange/SinkHandleTest.java | 4 ++--
.../iotdb/db/mpp/execution/operator/OperatorMemoryTest.java | 10 ++++++----
.../org/apache/iotdb/tsfile/common/conf/TSFileConfig.java | 4 ++--
.../iotdb/tsfile/read/common/block/TsBlockBuilder.java | 2 +-
6 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 48f87a76a9..abbdda959a 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -392,7 +392,7 @@
# The max capacity of a TsBlock
# Datatype: int, Unit: byte
-# max_tsblock_size_in_bytes=1048576
+# max_tsblock_size_in_bytes=131072
# The max number of lines in a single TsBlock
# Datatype: int
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/memory/MemoryPool.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/memory/MemoryPool.java
index b247846e59..b3d2fc2ff8 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/memory/MemoryPool.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/memory/MemoryPool.java
@@ -154,6 +154,14 @@ public class MemoryPool {
bytesToReserve > 0L && bytesToReserve <= maxBytesPerFragmentInstance,
"bytes should be greater than zero while less than or equal to max bytes per fragment instance: %d",
bytesToReserve);
+ if (bytesToReserve > maxBytesCanReserve) {
+ LOGGER.warn(
+ "Cannot reserve {} bytes memory from MemoryPool for planNodeId{}",
+ bytesToReserve,
+ planNodeId);
+ throw new IllegalArgumentException(
+ "Query is aborted since it requests more memory than can be allocated.");
+ }
ListenableFuture<Void> result;
synchronized (this) {
@@ -164,6 +172,10 @@ public class MemoryPool {
.getOrDefault(fragmentInstanceId, Collections.emptyMap())
.getOrDefault(planNodeId, 0L)
< bytesToReserve) {
+ LOGGER.debug(
+ "Blocked reserve request: {} bytes memory for planNodeId{}",
+ bytesToReserve,
+ planNodeId);
result =
MemoryReservationFuture.create(
queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve);
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandleTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandleTest.java
index 029eac1ccf..b11203dd11 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandleTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandleTest.java
@@ -48,7 +48,7 @@ public class SinkHandleTest {
@Test
public void testOneTimeNotBlockedSend() {
final String queryId = "q0";
- final long mockTsBlockSize = 1024L * 1024L;
+ final long mockTsBlockSize = 128 * 1024L;
final int numOfMockTsBlock = 1;
final TEndPoint remoteEndpoint =
new TEndPoint("remote", IoTDBDescriptor.getInstance().getConfig().getMppDataExchangePort());
@@ -186,7 +186,7 @@ public class SinkHandleTest {
@Test
public void testMultiTimesBlockedSend() {
final String queryId = "q0";
- final long mockTsBlockSize = 1024L * 1024L;
+ final long mockTsBlockSize = 128 * 1024L;
final int numOfMockTsBlock = 1;
final TEndPoint remoteEndpoint =
new TEndPoint("remote", IoTDBDescriptor.getInstance().getConfig().getMppDataExchangePort());
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
index b9d72485f8..69d6b2382c 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
@@ -1251,10 +1251,12 @@ public class OperatorMemoryTest {
typeProvider);
expectedMaxReturnSize =
- maxTsBlockLineNumber
- * (TimeColumn.SIZE_IN_BYTES_PER_POSITION
- + 512 * Byte.BYTES
- + LongColumn.SIZE_IN_BYTES_PER_POSITION);
+ Math.min(
+ DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES,
+ maxTsBlockLineNumber
+ * (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ + 512 * Byte.BYTES
+ + LongColumn.SIZE_IN_BYTES_PER_POSITION));
expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 6dbd28a67e..a1a5b22f88 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -151,8 +151,8 @@ public class TSFileConfig implements Serializable {
/** The amount of data iterate each time */
private int batchSize = 1000;
- /** Maximum capacity of a TsBlock */
- private int maxTsBlockSizeInBytes = 1024 * 1024;
+ /** Maximum capacity of a TsBlock, allow up to two pages. */
+ private int maxTsBlockSizeInBytes = 128 * 1024;
/** Maximum number of lines in a single TsBlock */
private int maxTsBlockLineNumber = 1000;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
index d4152d4ddb..c309835a09 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
@@ -259,7 +259,7 @@ public class TsBlockBuilder {
}
public boolean isFull() {
- return declaredPositions == MAX_LINE_NUMBER || tsBlockBuilderStatus.isFull();
+ return declaredPositions >= MAX_LINE_NUMBER || tsBlockBuilderStatus.isFull();
}
public boolean isEmpty() {