You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/11/24 02:38:03 UTC
[incubator-doris] branch master updated: [feat-opt](fe-config) Add tablets number limit to void wrong usage (#7025)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new d3c020b [feat-opt](fe-config) Add tablets number limit to void wrong usage (#7025)
d3c020b is described below
commit d3c020b3cb131d765018a168c1b8c6e304bf1626
Author: tianhui5 <82...@qq.com>
AuthorDate: Wed Nov 24 10:37:54 2021 +0800
[feat-opt](fe-config) Add tablets number limit to void wrong usage (#7025)
1. Add new FE config `default_db_replica_quota_size`
2. Check replica quota after create table/partition
---
docs/en/administrator-guide/config/fe_config.md | 17 +++++++++++
docs/zh-CN/administrator-guide/config/fe_config.md | 17 +++++++++++
.../java/org/apache/doris/catalog/Catalog.java | 35 ++++++++++++++++++++++
.../java/org/apache/doris/catalog/Database.java | 5 ++--
.../main/java/org/apache/doris/common/Config.java | 6 ++++
.../java/org/apache/doris/common/FeConstants.java | 1 -
6 files changed, 77 insertions(+), 4 deletions(-)
diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md
index 4961287..b2bb019 100644
--- a/docs/en/administrator-guide/config/fe_config.md
+++ b/docs/en/administrator-guide/config/fe_config.md
@@ -249,6 +249,23 @@ View configuration
show data （Detail：HELP SHOW DATA）
```
+### default_db_replica_quota_size
+
+Default: 1073741824
+
+IsMutable：true
+
+MasterOnly：true
+
+Used to set the default database replica quota. To set the quota size of a single database, you can use:
+
+```
+Set the database replica quota
+ALTER DATABASE db_name SET REPLICA QUOTA quota;
+View configuration
+show data （Detail：HELP SHOW DATA）
+```
+
### enable_batch_delete_by_default
Default：false
diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md
index 09c5c44..ce4b7a7 100644
--- a/docs/zh-CN/administrator-guide/config/fe_config.md
+++ b/docs/zh-CN/administrator-guide/config/fe_config.md
@@ -243,6 +243,23 @@ ALTER DATABASE db_name SET DATA QUOTA quota;
show data （其他用法：HELP SHOW DATA）
```
+### default_db_replica_quota_size
+
+默认值：1073741824
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：true
+
+用于设置默认数据库Replica数量配额大小，设置单个数据库配额大小可以使用：
+
+```
+设置数据库Replica数量配额
+ALTER DATABASE db_name SET REPLICA QUOTA quota;
+查看配置
+show data （其他用法：HELP SHOW DATA）
+```
+
### enable_batch_delete_by_default
默认值：false
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 6cf0aca..b56393b 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -3255,6 +3255,16 @@ public class Catalog {
DataProperty dataProperty = singlePartitionDesc.getPartitionDataProperty();
Preconditions.checkNotNull(dataProperty);
+ // check replica quota if this operation done
+ long indexNum = indexIdToMeta.size();
+ long bucketNum = distributionInfo.getBucketNum();
+ long replicaNum = singlePartitionDesc.getReplicaAlloc().getTotalReplicaNum();
+ long totalReplicaNum = indexNum * bucketNum * replicaNum;
+ if (totalReplicaNum >= db.getReplicaQuotaLeftWithLock()) {
+ throw new DdlException("Database " + db.getFullName() + " table " + tableName
+ + " add partition increasing " + totalReplicaNum
+ + " of replica exceeds quota[" + db.getReplicaQuota() + "]");
+ }
Set<Long> tabletIdSet = new HashSet<Long>();
try {
long partitionId = getNextId();
@@ -3837,6 +3847,17 @@ public class Catalog {
// use table name as partition name
String partitionName = tableName;
long partitionId = partitionNameToId.get(partitionName);
+
+ // check replica quota if this operation done
+ long indexNum = olapTable.getIndexIdToMeta().size();
+ long bucketNum = distributionInfo.getBucketNum();
+ long replicaNum = partitionInfo.getReplicaAllocation(partitionId).getTotalReplicaNum();
+ long totalReplicaNum = indexNum * bucketNum * replicaNum;
+ if (totalReplicaNum >= db.getReplicaQuotaLeftWithLock()) {
+ throw new DdlException("Database " + db.getFullName() + " create unpartitioned table "
+ + tableName + " increasing " + totalReplicaNum
+ + " of replica exceeds quota[" + db.getReplicaQuota() + "]");
+ }
// create partition
Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(),
olapTable.getId(), olapTable.getBaseIndexId(),
@@ -3872,6 +3893,20 @@ public class Catalog {
throw new DdlException(e.getMessage());
}
+ // check replica quota if this operation done
+ long totalReplicaNum = 0;
+ for (Map.Entry<String, Long> entry : partitionNameToId.entrySet()) {
+ long indexNum = olapTable.getIndexIdToMeta().size();
+ long bucketNum = distributionInfo.getBucketNum();
+ long replicaNum = partitionInfo.getReplicaAllocation(entry.getValue()).getTotalReplicaNum();
+ totalReplicaNum += indexNum * bucketNum * replicaNum;
+ }
+ if (totalReplicaNum >= db.getReplicaQuotaLeftWithLock()) {
+ throw new DdlException("Database " + db.getFullName() + " create table "
+ + tableName + " increasing " + totalReplicaNum
+ + " of replica exceeds quota[" + db.getReplicaQuota() + "]");
+ }
+
// this is a 2-level partitioned tables
for (Map.Entry<String, Long> entry : partitionNameToId.entrySet()) {
DataProperty dataProperty = partitionInfo.getDataProperty(entry.getValue());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 2e2bab2..9699c8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -23,7 +23,6 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
-import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
@@ -120,7 +119,7 @@ public class Database extends MetaObject implements Writable {
this.nameToTable = Maps.newConcurrentMap();
this.lowerCaseToTableName = Maps.newConcurrentMap();
this.dataQuotaBytes = Config.default_db_data_quota_bytes;
- this.replicaQuotaSize = FeConstants.default_db_replica_quota_size;
+ this.replicaQuotaSize = Config.default_db_replica_quota_size;
this.dbState = DbState.NORMAL;
this.attachDbName = "";
this.clusterName = "";
@@ -667,7 +666,7 @@ public class Database extends MetaObject implements Writable {
if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_81) {
replicaQuotaSize = in.readLong();
} else {
- replicaQuotaSize = FeConstants.default_db_replica_quota_size;
+ replicaQuotaSize = Config.default_db_replica_quota_size;
}
if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_105) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index bd8f152..2334138 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1394,6 +1394,12 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static long default_db_data_quota_bytes = 1024L * 1024 * 1024 * 1024 * 1024L; // 1PB
+ /**
+ * Used to set default db replica quota num.
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static long default_db_replica_quota_size = 1024 * 1024 * 1024;
+
/*
* Maximum percentage of data that can be filtered (due to reasons such as data is irregularly)
* The default value is 0.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index 49806fc..54b755c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -30,7 +30,6 @@ public class FeConstants {
*/
public static int shortkey_max_column_count = 3;
public static int shortkey_maxsize_bytes = 36;
- public static long default_db_replica_quota_size = 1024 * 1024 * 1024;
public static int heartbeat_interval_second = 5;
public static int checkpoint_interval_second = 60; // 1 minutes
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org