You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/03/20 03:30:29 UTC
[iotdb] branch data-region-per-datanode created (now fe19931657)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a change to branch data-region-per-datanode
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at fe19931657 Finish
This branch includes the following new commits:
new fe19931657 Finish
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Finish
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch data-region-per-datanode
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fe1993165780cfe2a8a83a7355f7c41f68c299fa
Author: YongzaoDan <53...@qq.com>
AuthorDate: Mon Mar 20 11:30:18 2023 +0800
Finish
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +++++-----
.../iotdb/confignode/conf/ConfigNodeDescriptor.java | 4 ++--
.../confignode/manager/ClusterSchemaManager.java | 19 +++++++++----------
.../iotdb/confignode/manager/ConfigManager.java | 6 +++---
docs/UserGuide/Cluster/Cluster-Maintenance.md | 2 +-
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 2 +-
.../it/cluster/IoTDBClusterNodeGetterIT.java | 4 ++--
.../confignode/it/utils/ConfigNodeTestUtils.java | 2 +-
.../assembly/resources/conf/iotdb-common.properties | 6 +++---
.../db/mpp/common/header/ColumnHeaderConstant.java | 2 +-
.../execution/config/metadata/ShowVariablesTask.java | 4 ++--
thrift-confignode/src/main/thrift/confignode.thrift | 2 +-
12 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 0a725b42b5..8e963ee0c3 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -101,7 +101,7 @@ public class ConfigNodeConfig {
private int defaultDataRegionGroupNumPerDatabase = 2;
/** The maximum number of DataRegions expected to be managed by each DataNode. */
- private double dataRegionPerProcessor = 1.0;
+ private double dataRegionPerDataNode = 5.0;
/** RegionGroup allocate policy. */
private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
@@ -546,12 +546,12 @@ public class ConfigNodeConfig {
this.dataRegionConsensusProtocolClass = dataRegionConsensusProtocolClass;
}
- public double getDataRegionPerProcessor() {
- return dataRegionPerProcessor;
+ public double getDataRegionPerDataNode() {
+ return dataRegionPerDataNode;
}
- public void setDataRegionPerProcessor(double dataRegionPerProcessor) {
- this.dataRegionPerProcessor = dataRegionPerProcessor;
+ public void setDataRegionPerDataNode(double dataRegionPerDataNode) {
+ this.dataRegionPerDataNode = dataRegionPerDataNode;
}
public RegionBalancer.RegionGroupAllocatePolicy getRegionGroupAllocatePolicy() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index db7b8c0a11..ef1536776d 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -250,11 +250,11 @@ public class ConfigNodeDescriptor {
"default_data_region_group_num_per_database",
String.valueOf(conf.getDefaultDataRegionGroupNumPerDatabase()).trim())));
- conf.setDataRegionPerProcessor(
+ conf.setDataRegionPerDataNode(
Double.parseDouble(
properties
.getProperty(
- "data_region_per_processor", String.valueOf(conf.getDataRegionPerProcessor()))
+ "data_region_per_data_node", String.valueOf(conf.getDataRegionPerDataNode()))
.trim()));
try {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
index 7c6295722f..89d93a6fa0 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
@@ -103,7 +103,7 @@ public class ClusterSchemaManager {
private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode();
- private static final double DATA_REGION_PER_PROCESSOR = CONF.getDataRegionPerProcessor();
+ private static final double DATA_REGION_PER_DATA_NODE = CONF.getDataRegionPerDataNode();
private final IManager configManager;
private final ClusterSchemaInfo clusterSchemaInfo;
@@ -376,7 +376,6 @@ public class ClusterSchemaManager {
}
int dataNodeNum = getNodeManager().getRegisteredDataNodeCount();
- int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount();
int databaseNum = databaseSchemaMap.size();
for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
@@ -389,8 +388,8 @@ public class ClusterSchemaManager {
AdjustMaxRegionGroupNumPlan adjustMaxRegionGroupNumPlan = new AdjustMaxRegionGroupNumPlan();
for (TDatabaseSchema databaseSchema : databaseSchemaMap.values()) {
try {
- // Adjust maxSchemaRegionGroupNum for each StorageGroup.
- // All StorageGroups share the DataNodes equally.
+ // Adjust maxSchemaRegionGroupNum for each Database.
+ // All Databases share the DataNodes equally.
// The allocated SchemaRegionGroups will not be shrunk.
int allocatedSchemaRegionGroupCount;
try {
@@ -415,8 +414,8 @@ public class ClusterSchemaManager {
databaseSchema.getName(),
maxSchemaRegionGroupNum);
- // Adjust maxDataRegionGroupNum for each StorageGroup.
- // All StorageGroups divide the total cpu cores equally.
+ // Adjust maxDataRegionGroupNum for each Database.
+ // All Databases share the DataNodes equally.
// The allocated DataRegionGroups will not be shrunk.
int allocatedDataRegionGroupCount =
getPartitionManager()
@@ -424,8 +423,8 @@ public class ClusterSchemaManager {
int maxDataRegionGroupNum =
calcMaxRegionGroupNum(
databaseSchema.getMinDataRegionGroupNum(),
- DATA_REGION_PER_PROCESSOR,
- totalCpuCoreNum,
+ DATA_REGION_PER_DATA_NODE,
+ dataNodeNum,
databaseNum,
databaseSchema.getDataReplicationFactor(),
allocatedDataRegionGroupCount);
@@ -447,7 +446,7 @@ public class ClusterSchemaManager {
int minRegionGroupNum,
double resourceWeight,
int resource,
- int storageGroupNum,
+ int databaseNum,
int replicationFactor,
int allocatedRegionGroupCount) {
return Math.max(
@@ -460,7 +459,7 @@ public class ClusterSchemaManager {
Math.ceil(
// The maxRegionGroupNum of the current StorageGroup is expected to be:
// (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor)
- resourceWeight * resource / (double) (storageGroupNum * replicationFactor)),
+ resourceWeight * resource / (double) (databaseNum * replicationFactor)),
// The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount
allocatedRegionGroupCount));
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index d92883c14f..924cf9ef1e 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -468,7 +468,7 @@ public class ConfigManager implements IManager {
clusterParameters.setTimePartitionInterval(CONF.getTimePartitionInterval());
clusterParameters.setDataReplicationFactor(CONF.getDataReplicationFactor());
clusterParameters.setSchemaReplicationFactor(CONF.getSchemaReplicationFactor());
- clusterParameters.setDataRegionPerProcessor(CONF.getDataRegionPerProcessor());
+ clusterParameters.setDataRegionPerDataNode(CONF.getDataRegionPerDataNode());
clusterParameters.setSchemaRegionPerDataNode(CONF.getSchemaRegionPerDataNode());
clusterParameters.setDiskSpaceWarningThreshold(COMMON_CONF.getDiskSpaceWarningThreshold());
clusterParameters.setReadConsistencyLevel(CONF.getReadConsistencyLevel());
@@ -1077,8 +1077,8 @@ public class ConfigManager implements IManager {
if (clusterParameters.getSchemaRegionPerDataNode() != CONF.getSchemaRegionPerDataNode()) {
return errorStatus.setMessage(errorPrefix + "schema_region_per_data_node" + errorSuffix);
}
- if (clusterParameters.getDataRegionPerProcessor() != CONF.getDataRegionPerProcessor()) {
- return errorStatus.setMessage(errorPrefix + "data_region_per_processor" + errorSuffix);
+ if (clusterParameters.getDataRegionPerDataNode() != CONF.getDataRegionPerDataNode()) {
+ return errorStatus.setMessage(errorPrefix + "data_region_per_data_node" + errorSuffix);
}
if (!clusterParameters.getReadConsistencyLevel().equals(CONF.getReadConsistencyLevel())) {
diff --git a/docs/UserGuide/Cluster/Cluster-Maintenance.md b/docs/UserGuide/Cluster/Cluster-Maintenance.md
index f10a4ed906..8c3ab7c809 100644
--- a/docs/UserGuide/Cluster/Cluster-Maintenance.md
+++ b/docs/UserGuide/Cluster/Cluster-Maintenance.md
@@ -44,7 +44,7 @@ IoTDB> show variables
| DefaultTTL(ms)| 9223372036854775807|
| ReadConsistencyLevel| strong|
| SchemaRegionPerDataNode| 1.0|
-| DataRegionPerProcessor| 1.0|
+| DataRegionPerDataNode| 5.0|
| LeastDataRegionGroupNum| 5|
| SeriesSlotNum| 10000|
| SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor|
diff --git a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
index ea7dd151d0..84bc486703 100644
--- a/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
+++ b/docs/zh/UserGuide/Cluster/Cluster-Maintenance.md
@@ -44,7 +44,7 @@ IoTDB> show variables
| DefaultTTL(ms)| 9223372036854775807|
| ReadConsistencyLevel| strong|
| SchemaRegionPerDataNode| 1.0|
-| DataRegionPerProcessor| 1.0|
+| DataRegionPerDataNode| 5.0|
| LeastDataRegionGroupNum| 5|
| SeriesSlotNum| 10000|
| SeriesSlotExecutorClass|org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor|
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
index 351d69a303..5694dbd25e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeGetterIT.java
@@ -162,8 +162,8 @@ public class IoTDBClusterNodeGetterIT {
expectedParameters.getSchemaReplicationFactor(),
clusterParameters.getSchemaReplicationFactor());
Assert.assertEquals(
- expectedParameters.getDataRegionPerProcessor(),
- clusterParameters.getDataRegionPerProcessor(),
+ expectedParameters.getDataRegionPerDataNode(),
+ clusterParameters.getDataRegionPerDataNode(),
0.01);
Assert.assertEquals(
expectedParameters.getSchemaRegionPerDataNode(),
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
index 735082c5d7..2bca1e15cd 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
@@ -208,7 +208,7 @@ public class ConfigNodeTestUtils {
clusterParameters.setTimePartitionInterval(604800000);
clusterParameters.setDataReplicationFactor(1);
clusterParameters.setSchemaReplicationFactor(1);
- clusterParameters.setDataRegionPerProcessor(1.0);
+ clusterParameters.setDataRegionPerDataNode(5.0);
clusterParameters.setSchemaRegionPerDataNode(1.0);
clusterParameters.setDiskSpaceWarningThreshold(0.05);
clusterParameters.setReadConsistencyLevel("strong");
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 0ac0c6f4ce..02bae820b8 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -103,7 +103,7 @@ cluster_name=defaultCluster
# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode.
# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema,
# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases.
-# Default is equal to the schema_replication_factor.
+# Default is equal to the schema_replication_factor to ensure each DataNode will have a SchemaRegionGroupLeader.
# Datatype: Double
# schema_region_per_data_node=1.0
@@ -122,11 +122,11 @@ cluster_name=defaultCluster
# default_data_region_group_num_per_database=2
# Only take effect when set data_region_group_extension_policy=AUTO.
-# This parameter is the maximum number of DataRegions expected to be managed by each processor.
+# This parameter is the maximum number of DataRegions expected to be managed by each DataNode.
# Notice: Since each Database requires at least two DataRegionGroups to manage its data,
# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases.
# Datatype: Double
-# data_region_per_processor=1.0
+# data_region_per_data_node=5.0
# Whether to enable the DataPartition inherit policy.
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
index e78060ce3a..2097a05478 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
@@ -114,7 +114,7 @@ public class ColumnHeaderConstant {
public static final String SERIES_SLOT_EXECUTOR_CLASS = "SeriesSlotExecutorClass";
public static final String DEFAULT_TTL = "DefaultTTL(ms)";
public static final String SCHEMA_REGION_PER_DATA_NODE = "SchemaRegionPerDataNode";
- public static final String DATA_REGION_PER_PROCESSOR = "DataRegionPerProcessor";
+ public static final String DATA_REGION_PER_DATA_NODE = "DataRegionPerDataNode";
public static final String READ_CONSISTENCY_LEVEL = "ReadConsistencyLevel";
public static final String DISK_SPACE_WARNING_THRESHOLD = "DiskSpaceWarningThreshold";
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
index 90e3ffd166..a43715fe9c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowVariablesTask.java
@@ -103,8 +103,8 @@ public class ShowVariablesTask implements IConfigTask {
new Binary(String.valueOf(clusterParameters.getSchemaRegionPerDataNode())));
buildTSBlock(
builder,
- new Binary(ColumnHeaderConstant.DATA_REGION_PER_PROCESSOR),
- new Binary(String.valueOf(clusterParameters.getDataRegionPerProcessor())));
+ new Binary(ColumnHeaderConstant.DATA_REGION_PER_DATA_NODE),
+ new Binary(String.valueOf(clusterParameters.getDataRegionPerDataNode())));
buildTSBlock(
builder,
new Binary(ColumnHeaderConstant.SERIES_SLOT_NUM),
diff --git a/thrift-confignode/src/main/thrift/confignode.thrift b/thrift-confignode/src/main/thrift/confignode.thrift
index fc866714f2..ac5ab69268 100644
--- a/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/thrift-confignode/src/main/thrift/confignode.thrift
@@ -333,7 +333,7 @@ struct TClusterParameters {
8: required i64 defaultTTL
9: required string readConsistencyLevel
10: required double schemaRegionPerDataNode
- 11: required double dataRegionPerProcessor
+ 11: required double dataRegionPerDataNode
12: required i32 seriesPartitionSlotNum
13: required string seriesPartitionExecutorClass
14: required double diskSpaceWarningThreshold