You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/04/07 08:57:33 UTC
[iotdb] 01/03: add config `into_operation_buffer_size_in_byte`
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/FixIntoMem
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cffbdab37b9788495a26c9f42c3b09412bf7bd2e
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Fri Apr 7 10:45:20 2023 +0800
add config `into_operation_buffer_size_in_byte`
---
docs/UserGuide/Reference/Common-Config-Manual.md | 10 ++++++++++
docs/zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++++++
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 +++++++++++++--
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 ++++++++++++
4 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md
index b7975023fe..403e39654c 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -1392,6 +1392,16 @@ Different configuration parameters take effect in the following three ways:
### SELECT-INTO
+* into\_operation\_buffer\_size\_in\_byte
+
+| Name | into\_operation\_buffer\_size\_in\_byte |
+| :---------: | :---------------------------------------------------------------------------------------------------------------------------------- |
+| Description | When the select-into statement is executed, the maximum memory occupied by the data to be written (unit: Byte) |
+| Type | int64 |
+| Default | 100MB |
+| Effective | hot-load |
+
+
* select\_into\_insert\_tablet\_plan\_row\_limit
| Name | select\_into\_insert\_tablet\_plan\_row\_limit |
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index 47ffa15a45..f9c5dc0869 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -1430,6 +1430,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
#### SELECT-INTO配置
+* into\_operation\_buffer\_size\_in\_byte
+
+| 名字 | into\_operation\_buffer\_size\_in\_byte |
+| :----------: | :-------------------------------------------------------------------- |
+| 描述 | 执行 select-into 语句时，待写入数据占用的最大内存（单位：Byte） |
+| 类型 | int64 |
+| 默认值 | 100MB |
+| 改后生效方式 | 热加载 |
+
* select\_into\_insert\_tablet\_plan\_row\_limit
| 名字 | select\_into\_insert\_tablet\_plan\_row\_limit |
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f92c093ddf..35bf672b4f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -683,6 +683,9 @@ public class IoTDBConfig {
*/
private long continuousQueryMinimumEveryInterval = 1000;
+ /** How much memory may be used in ONE SELECT INTO operation (in Byte). */
+ private long intoOperationBufferSizeInByte = 100 * 1024 * 1024L;
+
/**
* The maximum number of rows can be processed in insert-tablet-plan when executing select-into
* statements.
@@ -1901,14 +1904,22 @@ public class IoTDBConfig {
this.continuousQueryMinimumEveryInterval = minimumEveryInterval;
}
- public void setSelectIntoInsertTabletPlanRowLimit(int selectIntoInsertTabletPlanRowLimit) {
- this.selectIntoInsertTabletPlanRowLimit = selectIntoInsertTabletPlanRowLimit;
+ public long getIntoOperationBufferSizeInByte() {
+ return intoOperationBufferSizeInByte;
+ }
+
+ public void setIntoOperationBufferSizeInByte(long intoOperationBufferSizeInByte) {
+ this.intoOperationBufferSizeInByte = intoOperationBufferSizeInByte;
}
public int getSelectIntoInsertTabletPlanRowLimit() {
return selectIntoInsertTabletPlanRowLimit;
}
+ public void setSelectIntoInsertTabletPlanRowLimit(int selectIntoInsertTabletPlanRowLimit) {
+ this.selectIntoInsertTabletPlanRowLimit = selectIntoInsertTabletPlanRowLimit;
+ }
+
public int getIntoOperationExecutionThreadCount() {
return intoOperationExecutionThreadCount;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c1365a7685..3d94d1b88f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -907,6 +907,11 @@ public class IoTDBDescriptor {
// mqtt
loadMqttProps(properties);
+ conf.setIntoOperationBufferSizeInByte(
+ Long.parseLong(
+ properties.getProperty(
+ "into_operation_buffer_size_in_byte",
+ String.valueOf(conf.getIntoOperationBufferSizeInByte()))));
conf.setSelectIntoInsertTabletPlanRowLimit(
Integer.parseInt(
properties.getProperty(
@@ -1467,6 +1472,13 @@ public class IoTDBDescriptor {
properties.getProperty(
"merge_write_throughput_mb_per_sec",
Integer.toString(conf.getCompactionWriteThroughputMbPerSec()))));
+
+ // update select into operation max buffer size
+ conf.setIntoOperationBufferSizeInByte(
+ Long.parseLong(
+ properties.getProperty(
+ "into_operation_buffer_size_in_byte",
+ String.valueOf(conf.getIntoOperationBufferSizeInByte()))));
// update insert-tablet-plan's row limit for select-into
conf.setSelectIntoInsertTabletPlanRowLimit(
Integer.parseInt(