You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/09/20 08:05:37 UTC
[iotdb] branch master updated: [IOTDB-3656] mpp load with supporting modify MAX_PLAN_NODE_SIZE (#7367)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 935f530c00 [IOTDB-3656] mpp load with supporting modify MAX_PLAN_NODE_SIZE (#7367)
935f530c00 is described below
commit 935f530c00ed94c3d30c8dec2c530f76754a90d7
Author: yschengzi <87...@users.noreply.github.com>
AuthorDate: Tue Sep 20 16:05:29 2022 +0800
[IOTDB-3656] mpp load with supporting modify MAX_PLAN_NODE_SIZE (#7367)
---
.../src/assembly/resources/conf/iotdb-datanode.properties | 4 ++++
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +++++++---
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 15 +++++++++++++++
3 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 877bbfa861..e9892da209 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -99,6 +99,10 @@ target_config_nodes=127.0.0.1:22277
# Datatype: int
# connection_timeout_ms=20000
+# max plan node size, note that thrift max frame size must be larger than max plan node size.
+# Datatype: int
+# max_plan_node_size=536870912
+
# The maximum number of clients that can be idle for a node's InternalService.
# When the number of idle clients on a node exceeds this number, newly returned clients will be released
# Datatype: int
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 84095279e0..df25bd8ed3 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -759,9 +759,6 @@ public class IoTDBConfig {
*/
private long partitionInterval = 86400;
- /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */
- private long maxPlanNodeSize = 500 * 1048576L;
-
/**
* Level of TimeIndex, which records the start time and end time of TsFileResource. Currently,
* DEVICE_TIME_INDEX and FILE_TIME_INDEX are supported, and could not be changed after first set.
@@ -813,6 +810,9 @@ public class IoTDBConfig {
/** Unit: byte */
private int thriftMaxFrameSize = 536870912;
+ /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */
+ private int maxPlanNodeSize = thriftMaxFrameSize;
+
private int thriftDefaultBufferSize = RpcUtils.THRIFT_DEFAULT_BUF_CAPACITY;
/** time interval in minute for calculating query frequency. Unit: minute */
@@ -2445,6 +2445,10 @@ public class IoTDBConfig {
RpcTransportFactory.setThriftMaxFrameSize(this.thriftMaxFrameSize);
}
+ public void setMaxPlanNodeSize(int maxPlanNodeSize) {
+ this.maxPlanNodeSize = maxPlanNodeSize;
+ }
+
public int getThriftDefaultBufferSize() {
return thriftDefaultBufferSize;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 54a7d4b08b..1aae7bfe41 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -773,6 +773,21 @@ public class IoTDBDescriptor {
conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2);
}
+ conf.setMaxPlanNodeSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_plan_node_size", String.valueOf(conf.getThriftMaxFrameSize()))));
+
+ if (conf.getMaxPlanNodeSize() > conf.getThriftMaxFrameSize()) {
+ logger.warn(
+ String.format(
+ "MAX PLAN NODE SIZE %d can not be larger than THRIFT MAX FRAME SIZE %d, MAX PLAN NODE SIZE has been reset to %d",
+ conf.getMaxPlanNodeSize(),
+ conf.getThriftMaxFrameSize(),
+ conf.getThriftMaxFrameSize()));
+ conf.setMaxPlanNodeSize(conf.getThriftMaxFrameSize());
+ }
+
conf.setThriftDefaultBufferSize(
Integer.parseInt(
properties.getProperty(