You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/03/21 02:31:58 UTC
[iotdb] branch e646d3d87ed4 created (now 7a1c06fea6)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a change to branch e646d3d87ed4
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 7a1c06fea6 [IOTDB-5705] Replace data_region_per_processor by data_region_per_data_node (#9386)
This branch includes the following new commits:
new 7a1c06fea6 [IOTDB-5705] Replace data_region_per_processor by data_region_per_data_node (#9386)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-5705] Replace data_region_per_processor by data_region_per_data_node (#9386)
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch e646d3d87ed4
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7a1c06fea697f411c179fafd0aaeca3df622b7c2
Author: YongzaoDan <33...@users.noreply.github.com>
AuthorDate: Tue Mar 21 10:28:01 2023 +0800
[IOTDB-5705] Replace data_region_per_processor by data_region_per_data_node (#9386)
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +++++-----
.../iotdb/confignode/conf/ConfigNodeDescriptor.java | 4 ++--
.../confignode/manager/ClusterSchemaManager.java | 19 +++++++++----------
.../iotdb/confignode/manager/ConfigManager.java | 6 +++---
docs/UserGuide/Cluster/Cluster-Maintenance.md | 2 +-
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 2 +-
.../it/cluster/IoTDBClusterNodeGetterIT.java | 4 ++--
.../confignode/it/utils/ConfigNodeTestUtils.java | 2 +-
.../assembly/resources/conf/iotdb-common.properties | 6 +++---
.../db/mpp/common/header/ColumnHeaderConstant.java | 2 +-
.../execution/config/metadata/ShowVariablesTask.java | 4 ++--
thrift-confignode/src/main/thrift/confignode.thrift | 2 +-
12 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 71cac276b7..b4d0b77c7f 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -101,7 +101,7 @@ public class ConfigNodeConfig {
private int defaultDataRegionGroupNumPerDatabase = 2;
/** The maximum number of DataRegions expected to be managed by each DataNode. */
- private double dataRegionPerProcessor = 1.0;
+ private double dataRegionPerDataNode = 5.0;
/** RegionGroup allocate policy. */
private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
@@ -537,12 +537,12 @@ public class ConfigNodeConfig {
this.dataRegionConsensusProtocolClass = dataRegionConsensusProtocolClass;
}
- public double getDataRegionPerProcessor() {
- return dataRegionPerProcessor;
+ public double getDataRegionPerDataNode() {
+ return dataRegionPerDataNode;
}
- public void setDataRegionPerProcessor(double dataRegionPerProcessor) {
- this.dataRegionPerProcessor = dataRegionPerProcessor;
+ public void setDataRegionPerDataNode(double dataRegionPerDataNode) {
+ this.dataRegionPerDataNode = dataRegionPerDataNode;
}
public RegionBalancer.RegionGroupAllocatePolicy getRegionGroupAllocatePolicy() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index 6e508da618..5ff6618930 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -250,11 +250,11 @@ public class ConfigNodeDescriptor {
"default_data_region_group_num_per_database",
String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase()).trim())));
- conf.setDataRegionPerProcessor(
+ conf.setDataRegionPerDataNode(
Double.parseDouble(
properties
.getProperty(
- "data_region_per_processor", String.valueOf(conf.getDataRegionPerProcessor()))
+ "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode()))
.trim()));
try {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
index 8bd2458ed9..e6c0529255 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
@@ -103,7 +103,7 @@ public class ClusterSchemaManager {
private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode();
- private static final double DATA_REGION_PER_PROCESSOR = CONF.getDataRegionPerProcessor();
+ private static final double DATA_REGION_PER_DATA_NODE = CONF.getDataRegionPerDataNode();
private final IManager configManager;
private final ClusterSchemaInfo clusterSchemaInfo;
@@ -377,7 +377,6 @@ public class ClusterSchemaManager {
}
int dataNodeNum = getNodeManager().getRegisteredDataNodeCount();
- int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount();
int databaseNum = databaseSchemaMap.size();
for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
@@ -390,8 +389,8 @@ public class ClusterSchemaManager {
AdjustMaxRegionGroupNumPlan adjustMaxRegionGroupNumPlan = new AdjustMaxRegionGroupNumPlan();
for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
try {
- // Adjust maxSchemaRegionGroupNum for each StorageGroup.
- // All StorageGroups share the DataNodes equally.
+ // Adjust maxSchemaRegionGroupNum for each Database.
+ // All Databases share the DataNodes equally.
// The allocated SchemaRegionGroups will not be shrunk.
int allocatedSchemaRegionGroupCount;
try {
@@ -416,8 +415,8 @@ public class ClusterSchemaManager {
databaseSchema.getName(),
maxSchemaRegionGroupNum);
- // Adjust maxDataRegionGroupNum for each StorageGroup.
- // All StorageGroups divide the total cpu cores equally.
+ // Adjust maxDataRegionGroupNum for each Database.
+ // All Databases share the DataNodes equally.
// The allocated DataRegionGroups will not be shrunk.
int allocatedDataRegionGroupCount =
getPartitionManager()
@@ -425,8 +424,8 @@ public class ClusterSchemaManager {
int maxDataRegionGroupNum =
calcMaxRegionGroupNum(
databaseSchema.getMinDataRegionGroupNum(),
- DATA_REGION_PER_PROCESSOR,
- totalCpuCoreNum,
+ DATA_REGION_PER_DATA_NODE,
+ dataNodeNum,
databaseNum,
databaseSchema.getDataReplicationFactor(),
allocatedDataRegionGroupCount);
@@ -448,7 +447,7 @@ public class ClusterSchemaManager {
int minRegionGroupNum,
double resourceWeight,
int resource,
- int storageGroupNum,
+ int databaseNum,
int replicationFactor,
int allocatedRegionGroupCount) {
return Math.max(
@@ -461,7 +460,7 @@ public class ClusterSchemaManager {
Math.ceil(
// The maxRegionGroupNum of the current StorageGroup is expected to be:
// (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor)
- resourceWeight * resource / (double) (storageGroupNum * replicationFactor)),
+ resourceWeight * resource / (databaseNum * replicationFactor)),
// The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount
allocatedRegionGroupCount));
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 52aa33493f..6a4b8ddb15 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -456,7 +456,7 @@ public class ConfigManager implements IManager {
clusterParameters.setTimePartitionInterval(CONF.getTimePartitionInterval());
clusterParameters.setDataReplicationFactor(CONF.getDataReplicationFactor());
clusterParameters.setSchemaReplicationFactor(CONF.getSchemaReplicationFactor());
- clusterParameters.setDataRegionPerProcessor(CONF.getDataRegionPerProcessor());
+ clusterParameters.setDataRegionPerDataNode(CONF.getDataRegionPerDataNode());
clusterParameters.setSchemaRegionPerDataNode(CONF.getSchemaRegionPerDataNode());
clusterParameters.setDiskSpaceWarningThreshold(COMMON_CONF.getDiskSpaceWarningThreshold());
clusterParameters.setReadConsistencyLevel(CONF.getReadConsistencyLevel());
@@ -1071,8 +1071,8 @@ public class ConfigManager implements IManager {
if (clusterParameters.getSchemaRegionPerDataNode() != CONF.getSchemaRegionPerDataNode()) {
return errorStatus.setMessage(errorPrefix + "schema_region_per_data_node" + errorSuffix);
}
- if (clusterParameters.getDataRegionPerProcessor() != CONF.getDataRegionPerProcessor()) {
- return errorStatus.setMessage(errorPrefix + "data_region_per_processor" + errorSuffix);
+ if (clusterParameters.getDataRegionPerDataNode() != CONF.getDataRegionPerDataNode()) {
+ return errorStatus.setMessage(errorPrefix + "data_region_per_data_node" + errorSuffix);
}
if (!clusterParameters.getReadConsistencyLevel().equals(CONF.getReadConsistencyLevel())) {
diff --git a/docs/UserGuide/Cluster/Cluster-Maintenance.md b/docs/UserGuide/Cluster/Cluster-Maintenance.md
index f10a4ed906..8c3ab7c809 100644
--- a/docs/UserGuide/Cluster/Cluster-Maintenance.md
+++ b/docs/UserGuide/Cluster/Cluster-Maintenance.md
@@ -44,7 +44,7 @@ IoTDB> show variables
| DefaultTTL(ms)| 9223372036854775807|
| ReadConsistencyLevel| strong|
| SchemaRegionPerDataNode| 1.0|
-| DataRegionPerProcessor| 1.0|
+| DataRegionPerDataNode| 5.0|
| LeastDataRegionGroupNum| 5|
| SeriesSlotNum| 10000|
| SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor|
diff --git a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
index ea7dd151d0..84bc486703 100644
--- a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
+++ b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
@@ -44,7 +44,7 @@ IoTDB> show variables
| DefaultTTL(ms)| 9223372036854775807|
| ReadConsistencyLevel| strong|
| SchemaRegionPerDataNode| 1.0|
-| DataRegionPerProcessor| 1.0|
+| DataRegionPerDataNode| 5.0|
| LeastDataRegionGroupNum| 5|
| SeriesSlotNum| 10000|
| SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor|
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
index 351d69a303..5694dbd25e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
@@ -162,8 +162,8 @@ public class IoTDBClusterNodeGetterIT {
expectedParameters.getSchemaReplicationFactor(),
clusterParameters.getSchemaReplicationFactor());
Assert.assertEquals(
- expectedParameters.getDataRegionPerProcessor(),
- clusterParameters.getDataRegionPerProcessor(),
+ expectedParameters.getDataRegionPerDataNode(),
+ clusterParameters.getDataRegionPerDataNode(),
0.01);
Assert.assertEquals(
expectedParameters.getSchemaRegionPerDataNode(),
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
index 735082c5d7..2bca1e15cd 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
@@ -208,7 +208,7 @@ public class ConfigNodeTestUtils {
clusterParameters.setTimePartitionInterval(604800000);
clusterParameters.setDataReplicationFactor(1);
clusterParameters.setSchemaReplicationFactor(1);
- clusterParameters.setDataRegionPerProcessor(1.0);
+ clusterParameters.setDataRegionPerDataNode(5.0);
clusterParameters.setSchemaRegionPerDataNode(1.0);
clusterParameters.setDiskSpaceWarningThreshold(0.05);
clusterParameters.setReadConsistencyLevel("strong");
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 3f42740fa1..9f487ca068 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -103,7 +103,7 @@ cluster_name=defaultCluster
# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode.
# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema,
# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases.
-# Default is equal to the schema_replication_factor.
+# Default is equal to the schema_replication_factor to ensure each DataNode will have a SchemaRegionGroupLeader.
# Datatype: Double
# schema_region_per_data_node=1.0
@@ -122,11 +122,11 @@ cluster_name=defaultCluster
# default_data_region_group_num_per_database=2
# Only take effect when set data_region_group_extension_policy=AUTO.
-# This parameter is the maximum number of DataRegions expected to be managed by each processor.
+# This parameter is the maximum number of DataRegions expected to be managed by each DataNode.
# Notice: Since each Database requires at least two DataRegionGroups to manage its data,
# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases.
# Datatype: Double
-# data_region_per_processor=1.0
+# data_region_per_data_node=5.0
# Whether to enable the DataPartition inherit policy.
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
index baaf9a773c..4a11b2dd99 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
@@ -110,7 +110,7 @@ public class ColumnHeaderConstant {
public static final String SERIES_SLOT_EXECUTOR_CLASS = "SeriesSlotExecutorClass";
public static final String DEFAULT_TTL = "DefaultTTL(ms)";
public static final String SCHEMA_REGION_PER_DATA_NODE = "SchemaRegionPerDataNode";
- public static final String DATA_REGION_PER_PROCESSOR = "DataRegionPerProcessor";
+ public static final String DATA_REGION_PER_DATA_NODE = "DataRegionPerDataNode";
public static final String READ_CONSISTENCY_LEVEL = "ReadConsistencyLevel";
public static final String DISK_SPACE_WARNING_THRESHOLD = "DiskSpaceWarningThreshold";
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
index 90e3ffd166..a43715fe9c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
@@ -103,8 +103,8 @@ public class ShowVariablesTask implements IConfigTask {
new Binary(String.valueOf(clusterParameters.getSchemaRegionPerDataNode())));
buildTSBlock(
builder,
- new Binary(ColumnHeaderConstant.DATA_REGION_PER_PROCESSOR),
- new Binary(String.valueOf(clusterParameters.getDataRegionPerProcessor())));
+ new Binary(ColumnHeaderConstant.DATA_REGION_PER_DATA_NODE),
+ new Binary(String.valueOf(clusterParameters.getDataRegionPerDataNode())));
buildTSBlock(
builder,
new Binary(ColumnHeaderConstant.SERIES_SLOT_NUM),
diff --git a/thrift-confignode/src/main/thrift/confignode.thrift b/thrift-confignode/src/main/thrift/confignode.thrift
index b29b460d5f..d6858a6555 100644
--- a/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/thrift-confignode/src/main/thrift/confignode.thrift
@@ -332,7 +332,7 @@ struct TClusterParameters {
8: required i64 defaultTTL
9: required string readConsistencyLevel
10: required double schemaRegionPerDataNode
- 11: required double dataRegionPerProcessor
+ 11: required double dataRegionPerDataNode
12: required i32 seriesPartitionSlotNum
13: required string seriesPartitionExecutorClass
14: required double diskSpaceWarningThreshold