You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/05/08 06:26:12 UTC
[iotdb] branch master updated: Use two params to distinguish fsync delay in the wal sync mode and async mode (#9707)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 109b15434c Use two params to distinguish fsync delay in the wal sync mode and async mode (#9707)
109b15434c is described below
commit 109b15434c39c4f9bcbc6e564d9f128ecb86bb53
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Mon May 8 14:26:06 2023 +0800
Use two params to distinguish fsync delay in the wal sync mode and async mode (#9707)
---
docs/UserGuide/Reference/Common-Config-Manual.md | 29 ++++++++++------
docs/UserGuide/Reference/DataNode-Config-Manual.md | 4 +--
.../zh/UserGuide/Reference/Common-Config-Manual.md | 29 ++++++++++------
.../UserGuide/Reference/DataNode-Config-Manual.md | 4 +--
.../resources/conf/iotdb-common.properties | 15 +++++---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 +++++++++++++---------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 26 +++++++-------
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 9 ++++-
8 files changed, 98 insertions(+), 58 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md
index 562486bc03..964d14469b 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -1073,14 +1073,23 @@ Different configuration parameters take effect in the following three ways:
| Default | 0 |
| Effective | After restart system |
-* fsync\_wal\_delay\_in\_ms
+* wal\_async\_mode\_fsync\_delay\_in\_ms
-| Name | fsync\_wal\_delay\_in\_ms |
-|:-----------:|:--------------------------------------------------------------|
-| Description | Duration a wal flush operation will wait before calling fsync |
-| Type | int32 |
-| Default | 3 |
-| Effective | hot-load |
+| Name | wal\_async\_mode\_fsync\_delay\_in\_ms |
+|:-----------:|:--------------------------------------------------------------------------------|
+| Description | Duration a wal flush operation will wait before calling fsync in the async mode |
+| Type | int32 |
+| Default | 1000 |
+| Effective | hot-load |
+
+* wal\_sync\_mode\_fsync\_delay\_in\_ms
+
+| Name | wal\_sync\_mode\_fsync\_delay\_in\_ms |
+|:-----------:|:-------------------------------------------------------------------------------|
+| Description | Duration a wal flush operation will wait before calling fsync in the sync mode |
+| Type | int32 |
+| Default | 3 |
+| Effective | hot-load |
* wal\_buffer\_size\_in\_byte
@@ -1088,7 +1097,7 @@ Different configuration parameters take effect in the following three ways:
|:-----------:|:-----------------------------|
| Description | Buffer size of each wal node |
| Type | int32 |
-| Default | 16777216 |
+| Default | 33554432 |
| Effective | After restart system |
* wal\_buffer\_queue\_capacity
@@ -1097,7 +1106,7 @@ Different configuration parameters take effect in the following three ways:
|:-----------:|:-------------------------------------------|
| Description | Blocking queue capacity of each wal buffer |
| Type | int32 |
-| Default | 50 |
+| Default | 500 |
| Effective | After restart system |
* wal\_file\_size\_threshold\_in\_byte
@@ -1106,7 +1115,7 @@ Different configuration parameters take effect in the following three ways:
|:-----------:|:-------------------------------------|
| Description | Size threshold of each wal file |
| Type | int32 |
-| Default | 10485760 |
+| Default | 31457280 |
| Effective | hot-load |
* wal\_min\_effective\_info\_ratio
diff --git a/docs/UserGuide/Reference/DataNode-Config-Manual.md b/docs/UserGuide/Reference/DataNode-Config-Manual.md
index 5a71e0ec8b..1ad81b692d 100644
--- a/docs/UserGuide/Reference/DataNode-Config-Manual.md
+++ b/docs/UserGuide/Reference/DataNode-Config-Manual.md
@@ -311,9 +311,9 @@ The permission definitions are in ${IOTDB\_CONF}/conf/jmx.access.
| Default | data/datanode/consensus |
| Effective | After restarting system |
-* dn\_wal\_dir
+* dn\_wal\_dirs
-| Name | dn\_wal\_dir |
+| Name | dn\_wal\_dirs |
|:-----------:|:-------------------------------------------------------------------------|
| Description | Write Ahead Log storage path. It is recommended to use an absolute path. |
| Type | String |
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index a1a22cd8f0..f3b46cdd1e 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -1121,14 +1121,23 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
| 默认值 | 0 |
| 改后生效方式 | 重启服务生效 |
-* fsync\_wal\_delay\_in\_ms
+* wal\_async\_mode\_fsync\_delay\_in\_ms
-| 名字 | fsync\_wal\_delay\_in\_ms |
-|:------:|:--------------------------|
-| 描述 | 写前日志调用 fsync 前的等待时间 |
-| 类型 | int32 |
-| 默认值 | 3 |
-| 改后生效方式 | 热加载 |
+| 名字 | wal\_async\_mode\_fsync\_delay\_in\_ms |
+|:------:|:---------------------------------------|
+| 描述 | async 模式下写前日志调用 fsync 前的等待时间 |
+| 类型 | int32 |
+| 默认值 | 1000 |
+| 改后生效方式 | 热加载 |
+
+* wal\_sync\_mode\_fsync\_delay\_in\_ms
+
+| 名字 | wal\_sync\_mode\_fsync\_delay\_in\_ms |
+|:------:|:--------------------------------------|
+| 描述 | sync 模式下写前日志调用 fsync 前的等待时间 |
+| 类型 | int32 |
+| 默认值 | 3 |
+| 改后生效方式 | 热加载 |
* wal\_buffer\_size\_in\_byte
@@ -1136,7 +1145,7 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|:------:|:----------------------------|
| 描述 | 写前日志的 buffer 大小 |
| 类型 | int32 |
-| 默认值 | 16777216 |
+| 默认值 | 33554432 |
| 改后生效方式 | 重启服务生效 |
* wal\_buffer\_queue\_capacity
@@ -1145,7 +1154,7 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|:------:|:-----------------------------|
| 描述 | 写前日志阻塞队列大小上限 |
| 类型 | int32 |
-| 默认值 | 50 |
+| 默认值 | 500 |
| 改后生效方式 | 重启服务生效 |
* wal\_file\_size\_threshold\_in\_byte
@@ -1154,7 +1163,7 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|:------:|:-------------------------------------|
| 描述 | 写前日志文件封口阈值 |
| 类型 | int32 |
-| 默认值 | 10485760 |
+| 默认值 | 31457280 |
| 改后生效方式 | 热加载 |
* wal\_min\_effective\_info\_ratio
diff --git a/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md b/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md
index d212dbdec4..5e28430166 100644
--- a/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md
@@ -303,9 +303,9 @@ IoTDB DataNode 与 Standalone 模式共用一套配置文件，均位于 IoTDB
| 默认值 | data/datanode/consensus（Windows：data\\datanode\\consensus） |
| 改后生效方式 | 重启服务生效 |
-* dn\_wal\_dir
+* dn\_wal\_dirs
-| 名字 | dn\_wal\_dir |
+| 名字 | dn\_wal\_dirs |
|:------:|:---------------------------------------------------------------------|
| 描述 | IoTDB 写前日志存储路径，默认存放在和 sbin 目录同级的 data 目录下。相对路径的起始目录与操作系统相关，建议使用绝对路径。 |
| 类型 | String |
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index b1850ae6fc..323e57d00d 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -675,17 +675,24 @@ cluster_name=defaultCluster
# wal_mode=ASYNC
# Max number of wal nodes, each node corresponds to one wal directory
-# The default value 0 means the number is determined by the system.
+# This parameter is only valid in the standalone mode. IoTConsensus uses one wal per data region and RatisConsensus doesn't use wal.
+# The default value 0 means the number is determined by the system, the number is in the range of [data region num / 2, data region num].
# Notice: this value affects write performance significantly.
# For non-SSD disks, values between one third and half of databases number are recommended.
# Datatype: int
# max_wal_nodes_num=0
-# Duration a wal flush operation will wait before calling fsync
+# Duration a wal flush operation will wait before calling fsync in the async mode
# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists.
-# Notice: this value affects write performance significantly, values in the range of 0ms-2000ms are recommended.
+# Notice: this value affects write performance significantly, values in the range of 10ms-2000ms are recommended.
# Datatype: long
-# fsync_wal_delay_in_ms=1000
+# wal_async_mode_fsync_delay_in_ms=1000
+
+# Duration a wal flush operation will wait before calling fsync in the sync mode
+# A duration greater than 0 batches multiple wal fsync calls into one. This is useful when disks are slow or WAL write contention exists.
+# Notice: this value affects write performance significantly, values in the range of 0ms-10ms are recommended.
+# Datatype: long
+# wal_sync_mode_fsync_delay_in_ms=3
# Buffer size of each wal node
# If it's a value smaller than 0, use the default value 32 * 1024 * 1024 bytes (32MB).
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 2d0285c429..ae8ae5c1d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -184,15 +184,21 @@ public class IoTDBConfig {
/** Max number of wal nodes, each node corresponds to one wal directory */
private int maxWalNodesNum = 0;
- /** Duration a wal flush operation will wait before calling fsync. Unit: millisecond */
- private volatile long fsyncWalDelayInMs = 1000;
+ /**
+ * Duration a wal flush operation will wait before calling fsync in the async mode. Unit:
+ * millisecond
+ */
+ private volatile long walAsyncModeFsyncDelayInMs = 1_000;
+
+ /**
+ * Duration a wal flush operation will wait before calling fsync in the sync mode. Unit:
+ * millisecond
+ */
+ private volatile long walSyncModeFsyncDelayInMs = 3;
/** Buffer size of each wal node. Unit: byte */
private int walBufferSize = 32 * 1024 * 1024;
- /** Buffer entry size of each wal buffer. Unit: byte */
- private int walBufferEntrySize = 16 * 1024;
-
/** Blocking queue capacity of each wal buffer */
private int walBufferQueueCapacity = 500;
@@ -1721,28 +1727,28 @@ public class IoTDBConfig {
this.maxWalNodesNum = maxWalNodesNum;
}
- public long getFsyncWalDelayInMs() {
- return fsyncWalDelayInMs;
+ public long getWalAsyncModeFsyncDelayInMs() {
+ return walAsyncModeFsyncDelayInMs;
}
- void setFsyncWalDelayInMs(long fsyncWalDelayInMs) {
- this.fsyncWalDelayInMs = fsyncWalDelayInMs;
+ void setWalAsyncModeFsyncDelayInMs(long walAsyncModeFsyncDelayInMs) {
+ this.walAsyncModeFsyncDelayInMs = walAsyncModeFsyncDelayInMs;
}
- public int getWalBufferSize() {
- return walBufferSize;
+ public long getWalSyncModeFsyncDelayInMs() {
+ return walSyncModeFsyncDelayInMs;
}
- public void setWalBufferSize(int walBufferSize) {
- this.walBufferSize = walBufferSize;
+ public void setWalSyncModeFsyncDelayInMs(long walSyncModeFsyncDelayInMs) {
+ this.walSyncModeFsyncDelayInMs = walSyncModeFsyncDelayInMs;
}
- public int getWalBufferEntrySize() {
- return walBufferEntrySize;
+ public int getWalBufferSize() {
+ return walBufferSize;
}
- void setWalBufferEntrySize(int walBufferEntrySize) {
- this.walBufferEntrySize = walBufferEntrySize;
+ public void setWalBufferSize(int walBufferSize) {
+ this.walBufferSize = walBufferSize;
}
public int getWalBufferQueueCapacity() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 34614f1194..7b30821575 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1106,14 +1106,6 @@ public class IoTDBDescriptor {
conf.setWalBufferSize(walBufferSize);
}
- int walBufferEntrySize =
- Integer.parseInt(
- properties.getProperty(
- "wal_buffer_entry_size_in_byte", Integer.toString(conf.getWalBufferEntrySize())));
- if (walBufferEntrySize > 0) {
- conf.setWalBufferEntrySize(walBufferEntrySize);
- }
-
int walBufferQueueCapacity =
Integer.parseInt(
properties.getProperty(
@@ -1126,12 +1118,22 @@ public class IoTDBDescriptor {
}
private void loadWALHotModifiedProps(Properties properties) {
- long fsyncWalDelayInMs =
+ long walAsyncModeFsyncDelayInMs =
+ Long.parseLong(
+ properties.getProperty(
+ "wal_async_mode_fsync_delay_in_ms",
+ Long.toString(conf.getWalAsyncModeFsyncDelayInMs())));
+ if (walAsyncModeFsyncDelayInMs > 0) {
+ conf.setWalAsyncModeFsyncDelayInMs(walAsyncModeFsyncDelayInMs);
+ }
+
+ long walSyncModeFsyncDelayInMs =
Long.parseLong(
properties.getProperty(
- "fsync_wal_delay_in_ms", Long.toString(conf.getFsyncWalDelayInMs())));
- if (fsyncWalDelayInMs > 0) {
- conf.setFsyncWalDelayInMs(fsyncWalDelayInMs);
+ "wal_sync_mode_fsync_delay_in_ms",
+ Long.toString(conf.getWalSyncModeFsyncDelayInMs())));
+ if (walSyncModeFsyncDelayInMs > 0) {
+ conf.setWalSyncModeFsyncDelayInMs(walSyncModeFsyncDelayInMs);
}
long walFileSizeThreshold =
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java b/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java
index 41314d0dfe..b1cbfcf963 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.utils.MmapUtil;
import org.apache.iotdb.db.wal.exception.WALNodeClosedException;
import org.apache.iotdb.db.wal.io.WALMetaData;
import org.apache.iotdb.db.wal.utils.WALFileStatus;
+import org.apache.iotdb.db.wal.utils.WALMode;
import org.apache.iotdb.db.wal.utils.listener.WALFlushListener;
import org.slf4j.Logger;
@@ -182,7 +183,13 @@ public class WALBuffer extends AbstractWALBuffer {
WALEntry walEntry = null;
try {
// for better fsync performance, wait a while to enlarge write batch
- walEntry = walEntries.poll(config.getFsyncWalDelayInMs(), TimeUnit.MILLISECONDS);
+ if (config.getWalMode().equals(WALMode.ASYNC)) {
+ walEntry =
+ walEntries.poll(config.getWalAsyncModeFsyncDelayInMs(), TimeUnit.MILLISECONDS);
+ } else {
+ walEntry =
+ walEntries.poll(config.getWalSyncModeFsyncDelayInMs(), TimeUnit.MILLISECONDS);
+ }
} catch (InterruptedException e) {
logger.warn(
"Interrupted when waiting for taking WALEntry from blocking queue to serialize.");