You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2023/03/21 12:37:41 UTC
[iotdb] 04/06: [IOTDB-5705] Replace data_region_per_processor by data_region_per_data_node (#9386) (#9393)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rc/1.1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bf8a6c6ac70583f68214cbefc0f4680a582a3977
Author: YongzaoDan <33...@users.noreply.github.com>
AuthorDate: Tue Mar 21 12:25:15 2023 +0800
[IOTDB-5705] Replace data_region_per_processor by data_region_per_data_node (#9386) (#9393)
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +++++-----
.../iotdb/confignode/conf/ConfigNodeDescriptor.java | 4 ++--
.../confignode/manager/ClusterSchemaManager.java | 19 +++++++++----------
.../iotdb/confignode/manager/ConfigManager.java | 6 +++---
docs/UserGuide/Cluster/Cluster-Maintenance.md | 2 +-
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 2 +-
.../it/cluster/IoTDBClusterNodeGetterIT.java | 4 ++--
.../confignode/it/utils/ConfigNodeTestUtils.java | 2 +-
.../assembly/resources/conf/iotdb-common.properties | 6 +++---
.../db/mpp/common/header/ColumnHeaderConstant.java | 2 +-
.../execution/config/metadata/ShowVariablesTask.java | 4 ++--
thrift-confignode/src/main/thrift/confignode.thrift | 2 +-
12 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 71cac276b7..b4d0b77c7f 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -101,7 +101,7 @@ public class ConfigNodeConfig {
private int defaultDataRegionGroupNumPerDatabase = 2;
/** The maximum number of DataRegions expected to be managed by each DataNode. */
- private double dataRegionPerProcessor = 1.0;
+ private double dataRegionPerDataNode = 5.0;
/** RegionGroup allocate policy. */
private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
@@ -537,12 +537,12 @@ public class ConfigNodeConfig {
this.dataRegionConsensusProtocolClass = dataRegionConsensusProtocolClass;
}
- public double getDataRegionPerProcessor() {
- return dataRegionPerProcessor;
+ public double getDataRegionPerDataNode() {
+ return dataRegionPerDataNode;
}
- public void setDataRegionPerProcessor(double dataRegionPerProcessor) {
- this.dataRegionPerProcessor = dataRegionPerProcessor;
+ public void setDataRegionPerDataNode(double dataRegionPerDataNode) {
+ this.dataRegionPerDataNode = dataRegionPerDataNode;
}
public RegionBalancer.RegionGroupAllocatePolicy getRegionGroupAllocatePolicy() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index 6e508da618..5ff6618930 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -250,11 +250,11 @@ public class ConfigNodeDescriptor {
"default_data_region_group_num_per_database",
String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase()).trim())));
- conf.setDataRegionPerProcessor(
+ conf.setDataRegionPerDataNode(
Double.parseDouble(
properties
.getProperty(
- "data_region_per_processor", String.valueOf(conf.getDataRegionPerProcessor()))
+ "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode()))
.trim()));
try {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
index 8bd2458ed9..e6c0529255 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
@@ -103,7 +103,7 @@ public class ClusterSchemaManager {
private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode();
- private static final double DATA_REGION_PER_PROCESSOR = CONF.getDataRegionPerProcessor();
+ private static final double DATA_REGION_PER_DATA_NODE = CONF.getDataRegionPerDataNode();
private final IManager configManager;
private final ClusterSchemaInfo clusterSchemaInfo;
@@ -377,7 +377,6 @@ public class ClusterSchemaManager {
}
int dataNodeNum = getNodeManager().getRegisteredDataNodeCount();
- int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount();
int databaseNum = databaseSchemaMap.size();
for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
@@ -390,8 +389,8 @@ public class ClusterSchemaManager {
AdjustMaxRegionGroupNumPlan adjustMaxRegionGroupNumPlan = new AdjustMaxRegionGroupNumPlan();
for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
try {
- // Adjust maxSchemaRegionGroupNum for each StorageGroup.
- // All StorageGroups share the DataNodes equally.
+ // Adjust maxSchemaRegionGroupNum for each Database.
+ // All Databases share the DataNodes equally.
// The allocated SchemaRegionGroups will not be shrunk.
int allocatedSchemaRegionGroupCount;
try {
@@ -416,8 +415,8 @@ public class ClusterSchemaManager {
databaseSchema.getName(),
maxSchemaRegionGroupNum);
- // Adjust maxDataRegionGroupNum for each StorageGroup.
- // All StorageGroups divide the total cpu cores equally.
+ // Adjust maxDataRegionGroupNum for each Database.
+ // All Databases share the DataNodes equally.
// The allocated DataRegionGroups will not be shrunk.
int allocatedDataRegionGroupCount =
getPartitionManager()
@@ -425,8 +424,8 @@ public class ClusterSchemaManager {
int maxDataRegionGroupNum =
calcMaxRegionGroupNum(
databaseSchema.getMinDataRegionGroupNum(),
- DATA_REGION_PER_PROCESSOR,
- totalCpuCoreNum,
+ DATA_REGION_PER_DATA_NODE,
+ dataNodeNum,
databaseNum,
databaseSchema.getDataReplicationFactor(),
allocatedDataRegionGroupCount);
@@ -448,7 +447,7 @@ public class ClusterSchemaManager {
int minRegionGroupNum,
double resourceWeight,
int resource,
- int storageGroupNum,
+ int databaseNum,
int replicationFactor,
int allocatedRegionGroupCount) {
return Math.max(
@@ -461,7 +460,7 @@ public class ClusterSchemaManager {
Math.ceil(
// The maxRegionGroupNum of the current StorageGroup is expected to be:
// (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor)
- resourceWeight * resource / (double) (storageGroupNum * replicationFactor)),
+ resourceWeight * resource / (databaseNum * replicationFactor)),
// The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount
allocatedRegionGroupCount));
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 52aa33493f..6a4b8ddb15 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -456,7 +456,7 @@ public class ConfigManager implements IManager {
clusterParameters.setTimePartitionInterval(CONF.getTimePartitionInterval());
clusterParameters.setDataReplicationFactor(CONF.getDataReplicationFactor());
clusterParameters.setSchemaReplicationFactor(CONF.getSchemaReplicationFactor());
- clusterParameters.setDataRegionPerProcessor(CONF.getDataRegionPerProcessor());
+ clusterParameters.setDataRegionPerDataNode(CONF.getDataRegionPerDataNode());
clusterParameters.setSchemaRegionPerDataNode(CONF.getSchemaRegionPerDataNode());
clusterParameters.setDiskSpaceWarningThreshold(COMMON_CONF.getDiskSpaceWarningThreshold());
clusterParameters.setReadConsistencyLevel(CONF.getReadConsistencyLevel());
@@ -1071,8 +1071,8 @@ public class ConfigManager implements IManager {
if (clusterParameters.getSchemaRegionPerDataNode() != CONF.getSchemaRegionPerDataNode()) {
return errorStatus.setMessage(errorPrefix + "schema_region_per_data_node" + errorSuffix);
}
- if (clusterParameters.getDataRegionPerProcessor() != CONF.getDataRegionPerProcessor()) {
- return errorStatus.setMessage(errorPrefix + "data_region_per_processor" + errorSuffix);
+ if (clusterParameters.getDataRegionPerDataNode() != CONF.getDataRegionPerDataNode()) {
+ return errorStatus.setMessage(errorPrefix + "data_region_per_data_node" + errorSuffix);
}
if (!clusterParameters.getReadConsistencyLevel().equals(CONF.getReadConsistencyLevel())) {
diff --git a/docs/UserGuide/Cluster/Cluster-Maintenance.md b/docs/UserGuide/Cluster/Cluster-Maintenance.md
index f10a4ed906..8c3ab7c809 100644
--- a/docs/UserGuide/Cluster/Cluster-Maintenance.md
+++ b/docs/UserGuide/Cluster/Cluster-Maintenance.md
@@ -44,7 +44,7 @@ IoTDB> show variables
| DefaultTTL(ms)| 9223372036854775807|
| ReadConsistencyLevel| strong|
| SchemaRegionPerDataNode| 1.0|
-| DataRegionPerProcessor| 1.0|
+| DataRegionPerDataNode| 5.0|
| LeastDataRegionGroupNum| 5|
| SeriesSlotNum| 10000|
| SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor|
diff --git a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
index ea7dd151d0..84bc486703 100644
--- a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
+++ b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
@@ -44,7 +44,7 @@ IoTDB> show variables
| DefaultTTL(ms)| 9223372036854775807|
| ReadConsistencyLevel| strong|
| SchemaRegionPerDataNode| 1.0|
-| DataRegionPerProcessor| 1.0|
+| DataRegionPerDataNode| 5.0|
| LeastDataRegionGroupNum| 5|
| SeriesSlotNum| 10000|
| SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor|
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
index 351d69a303..5694dbd25e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
@@ -162,8 +162,8 @@ public class IoTDBClusterNodeGetterIT {
expectedParameters.getSchemaReplicationFactor(),
clusterParameters.getSchemaReplicationFactor());
Assert.assertEquals(
- expectedParameters.getDataRegionPerProcessor(),
- clusterParameters.getDataRegionPerProcessor(),
+ expectedParameters.getDataRegionPerDataNode(),
+ clusterParameters.getDataRegionPerDataNode(),
0.01);
Assert.assertEquals(
expectedParameters.getSchemaRegionPerDataNode(),
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
index 735082c5d7..2bca1e15cd 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
@@ -208,7 +208,7 @@ public class ConfigNodeTestUtils {
clusterParameters.setTimePartitionInterval(604800000);
clusterParameters.setDataReplicationFactor(1);
clusterParameters.setSchemaReplicationFactor(1);
- clusterParameters.setDataRegionPerProcessor(1.0);
+ clusterParameters.setDataRegionPerDataNode(5.0);
clusterParameters.setSchemaRegionPerDataNode(1.0);
clusterParameters.setDiskSpaceWarningThreshold(0.05);
clusterParameters.setReadConsistencyLevel("strong");
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 3f42740fa1..9f487ca068 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -103,7 +103,7 @@ cluster_name=defaultCluster
# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode.
# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema,
# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases.
-# Default is equal to the schema_replication_factor.
+# Default is equal to the schema_replication_factor to ensure each DataNode will have a SchemaRegionGroupLeader.
# Datatype: Double
# schema_region_per_data_node=1.0
@@ -122,11 +122,11 @@ cluster_name=defaultCluster
# default_data_region_group_num_per_database=2
# Only take effect when set data_region_group_extension_policy=AUTO.
-# This parameter is the maximum number of DataRegions expected to be managed by each processor.
+# This parameter is the maximum number of DataRegions expected to be managed by each DataNode.
# Notice: Since each Database requires at least two DataRegionGroups to manage its data,
# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases.
# Datatype: Double
-# data_region_per_processor=1.0
+# data_region_per_data_node=5.0
# Whether to enable the DataPartition inherit policy.
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
index baaf9a773c..4a11b2dd99 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
@@ -110,7 +110,7 @@ public class ColumnHeaderConstant {
public static final String SERIES_SLOT_EXECUTOR_CLASS = "SeriesSlotExecutorClass";
public static final String DEFAULT_TTL = "DefaultTTL(ms)";
public static final String SCHEMA_REGION_PER_DATA_NODE = "SchemaRegionPerDataNode";
- public static final String DATA_REGION_PER_PROCESSOR = "DataRegionPerProcessor";
+ public static final String DATA_REGION_PER_DATA_NODE = "DataRegionPerDataNode";
public static final String READ_CONSISTENCY_LEVEL = "ReadConsistencyLevel";
public static final String DISK_SPACE_WARNING_THRESHOLD = "DiskSpaceWarningThreshold";
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
index 90e3ffd166..a43715fe9c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
@@ -103,8 +103,8 @@ public class ShowVariablesTask implements IConfigTask {
new Binary(String.valueOf(clusterParameters.getSchemaRegionPerDataNode())));
buildTSBlock(
builder,
- new Binary(ColumnHeaderConstant.DATA_REGION_PER_PROCESSOR),
- new Binary(String.valueOf(clusterParameters.getDataRegionPerProcessor())));
+ new Binary(ColumnHeaderConstant.DATA_REGION_PER_DATA_NODE),
+ new Binary(String.valueOf(clusterParameters.getDataRegionPerDataNode())));
buildTSBlock(
builder,
new Binary(ColumnHeaderConstant.SERIES_SLOT_NUM),
diff --git a/thrift-confignode/src/main/thrift/confignode.thrift b/thrift-confignode/src/main/thrift/confignode.thrift
index b29b460d5f..d6858a6555 100644
--- a/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/thrift-confignode/src/main/thrift/confignode.thrift
@@ -332,7 +332,7 @@ struct TClusterParameters {
8: required i64 defaultTTL
9: required string readConsistencyLevel
10: required double schemaRegionPerDataNode
- 11: required double dataRegionPerProcessor
+ 11: required double dataRegionPerDataNode
12: required i32 seriesPartitionSlotNum
13: required string seriesPartitionExecutorClass
14: required double diskSpaceWarningThreshold