You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/01/13 00:58:36 UTC
[iotdb] branch rc/1.0.1 updated: Fix a regression issue that the max Region number is not calculated correctly (#8862)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch rc/1.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.0.1 by this push:
new 9c65c3a666 Fix a regression issue that the max Region number is not calculated correctly (#8862)
9c65c3a666 is described below
commit 9c65c3a666ddbcd820c3c33159b933adb3129ae2
Author: Zhang.Jinrui <xi...@gmail.com>
AuthorDate: Fri Jan 13 08:58:29 2023 +0800
Fix a regression issue that the max Region number is not calculated correctly (#8862)
Co-authored-by: YongzaoDan <33...@users.noreply.github.com>
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 4 +-
.../confignode/manager/ClusterSchemaManager.java | 79 +++++++++++-----------
.../manager/ClusterSchemaManagerTest.java | 38 +++++++++++
3 files changed, 81 insertions(+), 40 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 94e906a9ad..4be8758fde 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -95,10 +95,10 @@ public class ConfigNodeConfig {
private double dataRegionPerProcessor = 1.0;
/** The least number of SchemaRegionGroup for each Database. */
- private int leastSchemaRegionGroupNum = 1;
+ private volatile int leastSchemaRegionGroupNum = 1;
/** The least number of DataRegionGroup for each Database. */
- private int leastDataRegionGroupNum = 5;
+ private volatile int leastDataRegionGroupNum = 5;
/** RegionGroup allocate policy. */
private RegionBalancer.RegionGroupAllocatePolicy regionGroupAllocatePolicy =
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
index 861017f294..91746ace72 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterSchemaManager.java
@@ -97,9 +97,7 @@ public class ClusterSchemaManager {
private static final Logger LOGGER = LoggerFactory.getLogger(ClusterSchemaManager.class);
private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
- private static final int LEAST_SCHEMA_REGION_GROUP_NUM = CONF.getLeastSchemaRegionGroupNum();
private static final double SCHEMA_REGION_PER_DATA_NODE = CONF.getSchemaRegionPerDataNode();
- private static final int LEAST_DATA_REGION_GROUP_NUM = CONF.getLeastDataRegionGroupNum();
private static final double DATA_REGION_PER_PROCESSOR = CONF.getDataRegionPerProcessor();
private final IManager configManager;
@@ -316,6 +314,11 @@ public class ClusterSchemaManager {
// Get all StorageGroupSchemas
Map<String, TStorageGroupSchema> storageGroupSchemaMap =
getMatchedStorageGroupSchemasByName(getStorageGroupNames());
+ if (storageGroupSchemaMap.size() == 0) {
+ // Skip when there are no StorageGroups
+ return;
+ }
+
int dataNodeNum = getNodeManager().getRegisteredDataNodeCount();
int totalCpuCoreNum = getNodeManager().getTotalCpuCoreCount();
int storageGroupNum = storageGroupSchemaMap.size();
@@ -348,24 +351,13 @@ public class ClusterSchemaManager {
.getRegionGroupCount(
storageGroupSchema.getName(), TConsensusGroupType.SchemaRegion);
int maxSchemaRegionGroupNum =
- Math.max(
- // The least number of SchemaRegionGroup of each StorageGroup is specified
- // by parameter least_schema_region_group_num, which is currently unconfigurable.
- LEAST_SCHEMA_REGION_GROUP_NUM,
- Math.max(
- (int)
- // Use Math.ceil here to ensure that the maxSchemaRegionGroupNum
- // will be increased as long as the number of cluster DataNodes is increased
- Math.ceil(
- // The maxSchemaRegionGroupNum of the current StorageGroup
- // is expected to be:
- // (SCHEMA_REGION_PER_DATA_NODE * registerDataNodeNum) /
- // (createdStorageGroupNum * schemaReplicationFactor)
- SCHEMA_REGION_PER_DATA_NODE
- * dataNodeNum
- / storageGroupNum
- * storageGroupSchema.getSchemaReplicationFactor()),
- allocatedSchemaRegionGroupCount));
+ calcMaxRegionGroupNum(
+ CONF.getLeastSchemaRegionGroupNum(),
+ SCHEMA_REGION_PER_DATA_NODE,
+ dataNodeNum,
+ storageGroupNum,
+ storageGroupSchema.getSchemaReplicationFactor(),
+ allocatedSchemaRegionGroupCount);
LOGGER.info(
"[AdjustRegionGroupNum] The maximum number of SchemaRegionGroups for Database: {} is adjusted to: {}",
storageGroupSchema.getName(),
@@ -378,24 +370,13 @@ public class ClusterSchemaManager {
getPartitionManager()
.getRegionGroupCount(storageGroupSchema.getName(), TConsensusGroupType.DataRegion);
int maxDataRegionGroupNum =
- Math.max(
- // The least number of DataRegionGroup of each StorageGroup is specified
- // by parameter least_data_region_group_num.
- LEAST_DATA_REGION_GROUP_NUM,
- Math.max(
- (int)
- // Use Math.ceil here to ensure that the maxDataRegionGroupNum
- // will be increased as long as the number of cluster DataNodes is increased
- Math.ceil(
- // The maxDataRegionGroupNum of the current StorageGroup
- // is expected to be:
- // (DATA_REGION_PER_PROCESSOR * totalCpuCoreNum) /
- // (createdStorageGroupNum * dataReplicationFactor)
- DATA_REGION_PER_PROCESSOR
- * totalCpuCoreNum
- / storageGroupNum
- * storageGroupSchema.getDataReplicationFactor()),
- allocatedDataRegionGroupCount));
+ calcMaxRegionGroupNum(
+ CONF.getLeastDataRegionGroupNum(),
+ DATA_REGION_PER_PROCESSOR,
+ totalCpuCoreNum,
+ storageGroupNum,
+ storageGroupSchema.getDataReplicationFactor(),
+ allocatedDataRegionGroupCount);
LOGGER.info(
"[AdjustRegionGroupNum] The maximum number of DataRegionGroups for Database: {} is adjusted to: {}",
storageGroupSchema.getName(),
@@ -411,6 +392,28 @@ public class ClusterSchemaManager {
getConsensusManager().write(adjustMaxRegionGroupNumPlan);
}
+ public static int calcMaxRegionGroupNum(
+ int leastRegionGroupNum,
+ double resourceWeight,
+ int resource,
+ int storageGroupNum,
+ int replicationFactor,
+ int allocatedRegionGroupCount) {
+ return Math.max(
+ // The maxRegionGroupNum should be great or equal to the leastRegionGroupNum
+ leastRegionGroupNum,
+ Math.max(
+ (int)
+ // Use Math.ceil here to ensure that the maxRegionGroupNum
+ // will be increased as long as the number of cluster DataNodes is increased
+ Math.ceil(
+ // The maxRegionGroupNum of the current StorageGroup is expected to be:
+ // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor)
+ resourceWeight * resource / (double) (storageGroupNum * replicationFactor)),
+ // The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount
+ allocatedRegionGroupCount));
+ }
+
// ======================================================
// Leader scheduling interfaces
// ======================================================
diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java
new file mode 100644
index 0000000000..2af715eb1b
--- /dev/null
+++ b/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.manager;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ClusterSchemaManagerTest {
+
+ @Test
+ public void testCalcMaxRegionGroupNum() {
+
+ // The maxRegionGroupNum should be great or equal to the leastRegionGroupNum
+ Assert.assertEquals(100, ClusterSchemaManager.calcMaxRegionGroupNum(100, 1.0, 3, 1, 3, 0));
+
+ // The maxRegionGroupNum should be great or equal to the allocatedRegionGroupCount
+ Assert.assertEquals(100, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 6, 2, 3, 100));
+
+ // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor)
+ Assert.assertEquals(20, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 120, 2, 3, 5));
+ }
+}