You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2022/11/02 05:06:13 UTC
[iotdb] branch master updated: [IOTDB-4616] Add conf for ConfigNode OneCopy protocol (#7815)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dbfacd9c1c [IOTDB-4616] Add conf for ConfigNode OneCopy protocol (#7815)
dbfacd9c1c is described below
commit dbfacd9c1c85d61c658a151f1cd210ed41b8f44e
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Wed Nov 2 13:06:06 2022 +0800
[IOTDB-4616] Add conf for ConfigNode OneCopy protocol (#7815)
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 23 ++++--
.../confignode/conf/ConfigNodeDescriptor.java | 18 +++--
.../statemachine/PartitionRegionStateMachine.java | 2 +-
.../iotdb/confignode/manager/ConsensusManager.java | 2 +-
.../resources/conf/iotdb-common.properties | 91 ++++++++++++----------
5 files changed, 81 insertions(+), 55 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index f4e62c0e7b..49b39d1b58 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -156,6 +156,7 @@ public class ConfigNodeConfig {
private long dataRegionRatisSnapshotTriggerThreshold = 400000L;
private long partitionRegionRatisSnapshotTriggerThreshold = 400000L;
+ private long partitionRegionOneCopySnapshotTriggerThreshold = 400000L;
private long schemaRegionRatisSnapshotTriggerThreshold = 400000L;
/** RatisConsensus protocol, allow flushing Raft Log asynchronously */
@@ -169,7 +170,7 @@ public class ConfigNodeConfig {
private long partitionRegionRatisLogSegmentSizeMax = 24 * 1024 * 1024L;
private long schemaRegionRatisLogSegmentSizeMax = 24 * 1024 * 1024L;
- private long partitionRegionStandAloneLogSegmentSizeMax = 24 * 1024 * 1024L;
+ private long partitionRegionOneCopyLogSegmentSizeMax = 24 * 1024 * 1024L;
/** RatisConsensus protocol, flow control window for ratis grpc log appender */
private long dataRegionRatisGrpcFlowControlWindow = 4 * 1024 * 1024L;
@@ -619,6 +620,16 @@ public class ConfigNodeConfig {
partitionRegionRatisSnapshotTriggerThreshold;
}
+ public long getPartitionRegionOneCopySnapshotTriggerThreshold() {
+ return partitionRegionOneCopySnapshotTriggerThreshold;
+ }
+
+ public void setPartitionRegionOneCopySnapshotTriggerThreshold(
+ long partitionRegionOneCopySnapshotTriggerThreshold) {
+ this.partitionRegionOneCopySnapshotTriggerThreshold =
+ partitionRegionOneCopySnapshotTriggerThreshold;
+ }
+
public boolean isPartitionRegionRatisLogUnsafeFlushEnable() {
return partitionRegionRatisLogUnsafeFlushEnable;
}
@@ -701,13 +712,13 @@ public class ConfigNodeConfig {
this.schemaRegionRatisLogSegmentSizeMax = schemaRegionRatisLogSegmentSizeMax;
}
- public long getPartitionRegionStandAloneLogSegmentSizeMax() {
- return partitionRegionStandAloneLogSegmentSizeMax;
+ public long getPartitionRegionOneCopyLogSegmentSizeMax() {
+ return partitionRegionOneCopyLogSegmentSizeMax;
}
- public void setPartitionRegionStandAloneLogSegmentSizeMax(
- long partitionRegionStandAloneLogSegmentSizeMax) {
- this.partitionRegionStandAloneLogSegmentSizeMax = partitionRegionStandAloneLogSegmentSizeMax;
+ public void setPartitionRegionOneCopyLogSegmentSizeMax(
+ long partitionRegionOneCopyLogSegmentSizeMax) {
+ this.partitionRegionOneCopyLogSegmentSizeMax = partitionRegionOneCopyLogSegmentSizeMax;
}
public long getSchemaRegionRatisGrpcFlowControlWindow() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index 5310933a14..f31fb6cf42 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -334,6 +334,12 @@ public class ConfigNodeDescriptor {
"partition_region_ratis_snapshot_trigger_threshold",
String.valueOf(conf.getPartitionRegionRatisSnapshotTriggerThreshold()))));
+ conf.setPartitionRegionOneCopySnapshotTriggerThreshold(
+ Long.parseLong(
+ properties.getProperty(
+ "partition_region_one_copy_snapshot_trigger_threshold",
+ String.valueOf(conf.getPartitionRegionOneCopySnapshotTriggerThreshold()))));
+
conf.setSchemaRegionRatisSnapshotTriggerThreshold(
Long.parseLong(
properties.getProperty(
@@ -361,26 +367,26 @@ public class ConfigNodeDescriptor {
conf.setDataRegionRatisLogSegmentSizeMax(
Long.parseLong(
properties.getProperty(
- "data_region_ratis_log_segment_size_max",
+ "data_region_ratis_log_segment_size_max_in_byte",
String.valueOf(conf.getDataRegionRatisLogSegmentSizeMax()))));
conf.setPartitionRegionRatisLogSegmentSizeMax(
Long.parseLong(
properties.getProperty(
- "partition_region_ratis_log_segment_size_max",
+ "partition_region_ratis_log_segment_size_max_in_byte",
String.valueOf(conf.getPartitionRegionRatisLogSegmentSizeMax()))));
conf.setSchemaRegionRatisLogSegmentSizeMax(
Long.parseLong(
properties.getProperty(
- "schema_region_ratis_log_segment_size_max",
+ "schema_region_ratis_log_segment_size_max_in_byte",
String.valueOf(conf.getSchemaRegionRatisLogSegmentSizeMax()))));
- conf.setPartitionRegionStandAloneLogSegmentSizeMax(
+ conf.setPartitionRegionOneCopyLogSegmentSizeMax(
Long.parseLong(
properties.getProperty(
- "partition_region_standalone_log_segment_size_max",
- String.valueOf(conf.getPartitionRegionStandAloneLogSegmentSizeMax()))));
+ "partition_region_one_copy_log_segment_size_max_in_byte",
+ String.valueOf(conf.getPartitionRegionOneCopyLogSegmentSizeMax()))));
conf.setDataRegionRatisGrpcFlowControlWindow(
Long.parseLong(
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java
index f6b940d548..4fccc0d2e4 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java
@@ -64,7 +64,7 @@ public class PartitionRegionStateMachine
private int logFileId;
private static final String fileDir = CONF.getConsensusDir();
private static final String filePath = fileDir + File.separator + "log_inprogress_";
- private static final long FILE_MAX_SIZE = CONF.getPartitionRegionStandAloneLogSegmentSizeMax();
+ private static final long FILE_MAX_SIZE = CONF.getPartitionRegionOneCopyLogSegmentSizeMax();
private final TEndPoint currentNodeTEndPoint;
public PartitionRegionStateMachine(ConfigManager configManager, ConfigPlanExecutor executor) {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConsensusManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConsensusManager.java
index 9784e277e4..5523786a2c 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConsensusManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConsensusManager.java
@@ -86,7 +86,7 @@ public class ConsensusManager {
ConsensusConfig.newBuilder()
.setThisNode(
new TEndPoint(CONF.getInternalAddress(), CONF.getConsensusPort()))
- .setStorageDir("target" + java.io.File.separator + "standalone")
+ .setStorageDir("target" + java.io.File.separator + "one_copy")
.build(),
gid -> stateMachine)
.orElseThrow(
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index b613ac6cc9..59d2698cb8 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -130,7 +130,7 @@
# mlog_buffer_size=1048576
# The cycle when metadata log is periodically forced to be written to disk(in milliseconds)
-# If sync_mlog_period_in_ms = 0 it means force metadata log to be written to disk after each refreshment
+# If sync_mlog_period_in_ms=0 it means force metadata log to be written to disk after each refreshment
# Set this parameter to 0 may slow down the operation on slow disk.
# sync_mlog_period_in_ms=100
@@ -991,6 +991,13 @@ trigger_forward_mqtt_pool_size=4
### Region configuration
####################
+# ConfigNode consensus protocol type.
+# This parameter is unmodifiable after ConfigNode starts for the first time.
+# These consensus protocols are currently supported:
+# 1. org.apache.iotdb.consensus.ratis.RatisConsensus(Raft protocol)
+# Datatype: String
+# config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus
+
# SchemaRegion consensus protocol type.
# This parameter is unmodifiable after ConfigNode starts for the first time.
# These consensus protocols are currently supported:
@@ -1111,64 +1118,66 @@ trigger_forward_mqtt_pool_size=4
####################
# max payload size for a single log-sync-RPC from leader to follower
-# partition_region_ratis_log_appender_buffer_size_max = 4194304
-# schema_region_ratis_log_appender_buffer_size_max = 4194304
-# data_region_ratis_log_appender_buffer_size_max = 4194304
+# partition_region_ratis_log_appender_buffer_size_max=4194304
+# schema_region_ratis_log_appender_buffer_size_max=4194304
+# data_region_ratis_log_appender_buffer_size_max=4194304
-# trigger a snapshot when ratis_snapshot_trigger_threshold logs are written
-# partition_region_ratis_snapshot_trigger_threshold = 400000
-# schema_region_ratis_snapshot_trigger_threshold = 400000
-# data_region_ratis_snapshot_trigger_threshold = 400000
+# trigger a snapshot when snapshot_trigger_threshold logs are written
+# partition_region_ratis_snapshot_trigger_threshold=400000
+# schema_region_ratis_snapshot_trigger_threshold=400000
+# data_region_ratis_snapshot_trigger_threshold=400000
+# partition_region_one_copy_snapshot_trigger_threshold=400000
# allow flushing Raft Log asynchronously
-# partition_region_ratis_log_unsafe_flush_enable = false
-# schema_region_ratis_log_unsafe_flush_enable = false
-# data_region_ratis_log_unsafe_flush_enable = false
+# partition_region_ratis_log_unsafe_flush_enable=false
+# schema_region_ratis_log_unsafe_flush_enable=false
+# data_region_ratis_log_unsafe_flush_enable=false
-# max capacity of a single Raft Log segment (by default 24MB)
-# partition_region_ratis_log_segment_size_max = 25165824
-# schema_region_ratis_log_segment_size_max = 25165824
-# data_region_ratis_log_segment_size_max = 25165824
+# max capacity of a single Log segment file (in byte, by default 24MB)
+# partition_region_ratis_log_segment_size_max_in_byte=25165824
+# schema_region_ratis_log_segment_size_max_in_byte=25165824
+# data_region_ratis_log_segment_size_max_in_byte=25165824
+# partition_region_one_copy_log_segment_size_max_in_byte=25165824
# flow control window for ratis grpc log appender
-# partition_region_ratis_grpc_flow_control_window = 4194304
-# schema_region_ratis_grpc_flow_control_window = 4194304
-# data_region_ratis_grpc_flow_control_window = 4194304
+# partition_region_ratis_grpc_flow_control_window=4194304
+# schema_region_ratis_grpc_flow_control_window=4194304
+# data_region_ratis_grpc_flow_control_window=4194304
# min election timeout for leader election
-# partition_region_ratis_rpc_leader_election_timeout_min_ms = 2000
-# schema_region_ratis_rpc_leader_election_timeout_min_ms = 2000
-# data_region_ratis_rpc_leader_election_timeout_min_ms = 2000
+# partition_region_ratis_rpc_leader_election_timeout_min_ms=2000
+# schema_region_ratis_rpc_leader_election_timeout_min_ms=2000
+# data_region_ratis_rpc_leader_election_timeout_min_ms=2000
# max election timeout for leader election
-# partition_region_ratis_rpc_leader_election_timeout_max_ms = 4000
-# schema_region_ratis_rpc_leader_election_timeout_max_ms = 4000
-# data_region_ratis_rpc_leader_election_timeout_max_ms = 4000
+# partition_region_ratis_rpc_leader_election_timeout_max_ms=4000
+# schema_region_ratis_rpc_leader_election_timeout_max_ms=4000
+# data_region_ratis_rpc_leader_election_timeout_max_ms=4000
# ratis client retry threshold
-# partition_region_ratis_request_timeout_ms = 10000
-# schema_region_ratis_request_timeout_ms = 10000
-# data_region_ratis_request_timeout_ms = 10000
+# partition_region_ratis_request_timeout_ms=10000
+# schema_region_ratis_request_timeout_ms=10000
+# data_region_ratis_request_timeout_ms=10000
# currently we use exponential back-off retry policy for ratis
-# partition_region_ratis_max_retry_attempts = 10
-# partition_region_ratis_initial_sleep_time_ms = 100
-# partition_region_ratis_max_sleep_time_ms = 10000
-# schema_region_ratis_max_retry_attempts = 10
-# schema_region_ratis_initial_sleep_time_ms = 100
-# schema_region_ratis_max_sleep_time_ms = 10000
-# data_region_ratis_max_retry_attempts = 10
-# data_region_ratis_initial_sleep_time_ms = 100
-# data_region_ratis_max_sleep_time_ms = 10000
+# partition_region_ratis_max_retry_attempts=10
+# partition_region_ratis_initial_sleep_time_ms=100
+# partition_region_ratis_max_sleep_time_ms=10000
+# schema_region_ratis_max_retry_attempts=10
+# schema_region_ratis_initial_sleep_time_ms=100
+# schema_region_ratis_max_sleep_time_ms=10000
+# data_region_ratis_max_retry_attempts=10
+# data_region_ratis_initial_sleep_time_ms=100
+# data_region_ratis_max_sleep_time_ms=10000
# preserve certain logs when take snapshot and purge
-# partition_region_ratis_preserve_logs_num_when_purge = 1000
-# schema_region_ratis_preserve_logs_num_when_purge = 1000
-# data_region_ratis_preserve_logs_num_when_purge = 1000
+# partition_region_ratis_preserve_logs_num_when_purge=1000
+# schema_region_ratis_preserve_logs_num_when_purge=1000
+# data_region_ratis_preserve_logs_num_when_purge=1000
# first election timeout
-# ratis_first_election_timeout_min_ms = 50
-# ratis_first_election_timeout_max_ms = 150
+# ratis_first_election_timeout_min_ms=50
+# ratis_first_election_timeout_max_ms=150
####################
### Disk Monitor