[iotdb] 01/01: [IOTDB-5934] Optimize cluster partition policy (#9971)

[yo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yongzao pushed a commit to branch cherrypick_partition
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit ca4f974e4197ad641bac0639558e2b819edcb2c1
Author: YongzaoDan <33...@users.noreply.github.com>
AuthorDate: Tue May 30 00:56:45 2023 +0800

    [IOTDB-5934] Optimize cluster partition policy (#9971)
    
    * Update PartitionManager.java
    
    * finish
    
    * Fix IT bug
---
 .../apache/iotdb/confignode/conf/ConfigNodeConfig.java   |  4 ++--
 .../confignode/manager/partition/PartitionManager.java   | 16 +++++++++++++++-
 .../iotdb/confignode/it/utils/ConfigNodeTestUtils.java   |  2 +-
 .../src/assembly/resources/conf/iotdb-common.properties  |  2 +-
 4 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 22f2e86a196..a6a878da47d 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -68,8 +68,8 @@ public class ConfigNodeConfig {
   /** Default number of DataRegion replicas */
   private int dataReplicationFactor = 1;
 
-  /** Number of SeriesPartitionSlots per StorageGroup */
-  private int seriesSlotNum = 10000;
+  /** Number of SeriesPartitionSlots per Database */
+  private int seriesSlotNum = 1000;
 
   /** SeriesPartitionSlot executor class */
   private String seriesPartitionExecutorClass =
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java
index c90e510bbdc..71407eeff33 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java
@@ -208,6 +208,13 @@ public class PartitionManager {
     // by the number of SeriesPartitionSlots,
     // the number of serialized CreateSchemaPartitionReqs is acceptable.
     synchronized (this) {
+      // Here we should check again if the SchemaPartition
+      // has been created by other threads to improve concurrent performance
+      resp = (SchemaPartitionResp) getSchemaPartition(req);
+      if (resp.isAllPartitionsExist()) {
+        return resp;
+      }
+
       // Filter unassigned SchemaPartitionSlots
       Map<String, List<TSeriesPartitionSlot>> unassignedSchemaPartitionSlotsMap =
           partitionInfo.filterUnassignedSchemaPartitionSlots(req.getPartitionSlotsMap());
@@ -261,7 +268,7 @@ public class PartitionManager {
     resp = (SchemaPartitionResp) getSchemaPartition(req);
     if (!resp.isAllPartitionsExist()) {
       LOGGER.error(
-          "Lacked some SchemaPartition allocation result in the response of getOrCreateDataPartition method");
+          "Lacked some SchemaPartition allocation result in the response of getOrCreateSchemaPartition method");
       resp.setStatus(
           new TSStatus(TSStatusCode.LACK_PARTITION_ALLOCATION.getStatusCode())
               .setMessage("Lacked some SchemaPartition allocation result in the response"));
@@ -307,6 +314,13 @@ public class PartitionManager {
     // by the number of SeriesPartitionSlots,
     // the number of serialized CreateDataPartitionReqs is acceptable.
     synchronized (this) {
+      // Here we should check again if the DataPartition
+      // has been created by other threads to improve concurrent performance
+      resp = (DataPartitionResp) getDataPartition(req);
+      if (resp.isAllPartitionsExist()) {
+        return resp;
+      }
+
       // Filter unassigned DataPartitionSlots
       Map<String, Map<TSeriesPartitionSlot, TTimeSlotList>> unassignedDataPartitionSlotsMap =
           partitionInfo.filterUnassignedDataPartitionSlots(req.getPartitionSlotsMap());
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
index b3123838fcf..e1588956e2f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/utils/ConfigNodeTestUtils.java
@@ -201,7 +201,7 @@ public class ConfigNodeTestUtils {
         "org.apache.iotdb.consensus.simple.SimpleConsensus");
     clusterParameters.setSchemaRegionConsensusProtocolClass(
         "org.apache.iotdb.consensus.simple.SimpleConsensus");
-    clusterParameters.setSeriesPartitionSlotNum(10000);
+    clusterParameters.setSeriesPartitionSlotNum(1000);
     clusterParameters.setSeriesPartitionExecutorClass(
         "org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor");
     clusterParameters.setDefaultTTL(Long.MAX_VALUE);
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 75f1c972fb5..e9a62e60ae5 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -71,7 +71,7 @@ cluster_name=defaultCluster
 # And these parameters should be consistent within the ConfigNodeGroup.
 # Number of SeriesPartitionSlots per Database
 # Datatype: Integer
-# series_slot_num=10000
+# series_slot_num=1000
 
 # SeriesPartitionSlot executor class
 # These hashing algorithms are currently supported: