You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/07/05 08:28:45 UTC
[incubator-doris] branch master updated: [Dynamic Partition]Support
set replication number (#3965)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 913b2ca [Dynamic Partition]Support set replication number (#3965)
913b2ca is described below
commit 913b2caac47ce75724c0fe97814567b7c71437a7
Author: WingC <10...@qq.com>
AuthorDate: Sun Jul 5 03:28:38 2020 -0500
[Dynamic Partition]Support set replication number (#3965)
This CL mainly support set replication_num property in dynamic partition
table if dynamic_partition.replication_num is not set, the value is the
same as table's default replication_num.
---
docs/en/administrator-guide/dynamic-partition.md | 6 +++-
.../zh-CN/administrator-guide/dynamic-partition.md | 6 +++-
.../doris/analysis/ShowDynamicPartitionStmt.java | 1 +
.../doris/catalog/DynamicPartitionProperty.java | 9 +++++
.../doris/clone/DynamicPartitionScheduler.java | 6 +++-
.../java/org/apache/doris/common/ErrorCode.java | 6 +++-
.../doris/common/util/DynamicPartitionUtil.java | 34 +++++++++++--------
.../java/org/apache/doris/qe/ShowExecutor.java | 3 ++
.../doris/catalog/DynamicPartitionTableTest.java | 39 ++++++++++++++++++++++
9 files changed, 93 insertions(+), 17 deletions(-)
diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md
index f4c6a54..0fce2ac 100644
--- a/docs/en/administrator-guide/dynamic-partition.md
+++ b/docs/en/administrator-guide/dynamic-partition.md
@@ -108,7 +108,11 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`:
* `dynamic_partition.buckets`
The number of buckets corresponding to the dynamically created partitions.
-
+
+* `dynamic_partition.replication_num`
+
+ The replication number of dynamic partition.If not filled in, defaults to the number of table's replication number.
+
* `dynamic_partition.start_day_of_week`
When `time_unit` is` WEEK`, this parameter is used to specify the starting point of the week. The value ranges from 1 to 7. Where 1 is Monday and 7 is Sunday. The default is 1, which means that every week starts on Monday.
diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md
index 709c154..0e56a6c 100644
--- a/docs/zh-CN/administrator-guide/dynamic-partition.md
+++ b/docs/zh-CN/administrator-guide/dynamic-partition.md
@@ -106,7 +106,11 @@ under the License.
* `dynamic_partition.buckets`
动态创建的分区所对应的分桶数量。
-
+
+* `dynamic_partition.replication_num`
+
+ 动态创建的分区所对应的副本数量，如果不填写，则默认为该表创建时指定的副本数量。
+
* `dynamic_partition.start_day_of_week`
当 `time_unit` 为 `WEEK` 时，该参数用于指定每周的起始点。取值为 1 到 7。其中 1 表示周一，7 表示周日。默认为 1，即表示每周以周一为起始点。
diff --git a/fe/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java b/fe/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java
index 4e98bef..529da1d 100644
--- a/fe/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/ShowDynamicPartitionStmt.java
@@ -38,6 +38,7 @@ public class ShowDynamicPartitionStmt extends ShowStmt {
.addColumn(new Column("End", ScalarType.createVarchar(20)))
.addColumn(new Column("Prefix", ScalarType.createVarchar(20)))
.addColumn(new Column("Buckets", ScalarType.createVarchar(20)))
+ .addColumn(new Column("ReplicationNum", ScalarType.createVarchar(20)))
.addColumn(new Column("StartOf", ScalarType.createVarchar(20)))
.addColumn(new Column("LastUpdateTime", ScalarType.createVarchar(20)))
.addColumn(new Column("LastSchedulerTime", ScalarType.createVarchar(20)))
diff --git a/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
index 63ce513..8783acc 100644
--- a/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
+++ b/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
@@ -35,8 +35,10 @@ public class DynamicPartitionProperty {
public static final String START_DAY_OF_WEEK = "dynamic_partition.start_day_of_week";
public static final String START_DAY_OF_MONTH = "dynamic_partition.start_day_of_month";
public static final String TIME_ZONE = "dynamic_partition.time_zone";
+ public static final String REPLICATION_NUM = "dynamic_partition.replication_num";
public static final int MIN_START_OFFSET = Integer.MIN_VALUE;
+ public static final int NOT_SET_REPLICATION_NUM = -1;
private boolean exist;
@@ -49,6 +51,7 @@ public class DynamicPartitionProperty {
private StartOfDate startOfWeek;
private StartOfDate startOfMonth;
private TimeZone tz = TimeUtils.getSystemTimeZone();
+ private int replicationNum;
public DynamicPartitionProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
@@ -61,6 +64,7 @@ public class DynamicPartitionProperty {
this.end = Integer.parseInt(properties.get(END));
this.prefix = properties.get(PREFIX);
this.buckets = Integer.parseInt(properties.get(BUCKETS));
+ this.replicationNum = Integer.parseInt(properties.getOrDefault(REPLICATION_NUM, String.valueOf(NOT_SET_REPLICATION_NUM)));
createStartOfs(properties);
} else {
this.exist = false;
@@ -133,6 +137,10 @@ public class DynamicPartitionProperty {
return tz;
}
+ public int getReplicationNum() {
+ return replicationNum;
+ }
+
@Override
public String toString() {
String res = ",\n\"" + ENABLE + "\" = \"" + enable + "\"" +
@@ -141,6 +149,7 @@ public class DynamicPartitionProperty {
",\n\"" + START + "\" = \"" + start + "\"" +
",\n\"" + END + "\" = \"" + end + "\"" +
",\n\"" + PREFIX + "\" = \"" + prefix + "\"" +
+ ",\n\"" + REPLICATION_NUM + "\" = \"" + replicationNum + "\"" +
",\n\"" + BUCKETS + "\" = \"" + buckets + "\"";
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\"";
diff --git a/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 17d3b91..da3d844 100644
--- a/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ b/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -172,7 +172,11 @@ public class DynamicPartitionScheduler extends MasterDaemon {
// construct partition desc
PartitionKeyDesc partitionKeyDesc = new PartitionKeyDesc(Collections.singletonList(lowerValue), Collections.singletonList(upperValue));
HashMap<String, String> partitionProperties = new HashMap<>(1);
- partitionProperties.put("replication_num", String.valueOf(DynamicPartitionUtil.estimateReplicateNum(olapTable)));
+ if (dynamicPartitionProperty.getReplicationNum() == DynamicPartitionProperty.NOT_SET_REPLICATION_NUM) {
+ partitionProperties.put("replication_num", String.valueOf(olapTable.getDefaultReplicationNum()));
+ } else {
+ partitionProperties.put("replication_num", String.valueOf(dynamicPartitionProperty.getReplicationNum()));
+ }
String partitionName = dynamicPartitionProperty.getPrefix() + DynamicPartitionUtil.getFormattedPartitionName(
dynamicPartitionProperty.getTimeZone(), prevBorder, dynamicPartitionProperty.getTimeUnit());
SingleRangePartitionDesc rangePartitionDesc = new SingleRangePartitionDesc(true, partitionName,
diff --git a/fe/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/src/main/java/org/apache/doris/common/ErrorCode.java
index 7b416a3..186899c 100644
--- a/fe/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -232,7 +232,11 @@ public enum ErrorCode {
ERROR_DYNAMIC_PARTITION_PREFIX(5069, new byte[] {'4', '2', '0', '0', '0'},
"Invalid dynamic partition prefix: %s."),
ERR_OPERATION_DISABLED(5070, new byte[] {'4', '2', '0', '0', '0'},
- "Operation %s is disabled. %s");
+ "Operation %s is disabled. %s"),
+ ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO(5071, new byte[] {'4', '2', '0', '0', '0'},
+ "Dynamic partition replication num must greater than 0"),
+ ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT(5072, new byte[] {'4', '2', '0', '0', '0'},
+ "Invalid dynamic partition replication num: %s.");
ErrorCode(int code, byte[] sqlState, String errorMsg) {
this.code = code;
diff --git a/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index c9bdd83..8be5b2e 100644
--- a/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -23,7 +23,6 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
@@ -151,6 +150,19 @@ public class DynamicPartitionUtil {
}
}
+ private static void checkReplicationNum(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.REPLICATION_NUM);
+ }
+ try {
+ if (Integer.parseInt(val) <= 0) {
+ ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO);
+ }
+ } catch (NumberFormatException e) {
+ ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT, val);
+ }
+ }
+
public static boolean checkDynamicPartitionPropertiesExist(Map<String, String> properties) {
if (properties == null) {
return false;
@@ -161,6 +173,7 @@ public class DynamicPartitionUtil {
properties.containsKey(DynamicPartitionProperty.END) ||
properties.containsKey(DynamicPartitionProperty.PREFIX) ||
properties.containsKey(DynamicPartitionProperty.BUCKETS) ||
+ properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM) ||
properties.containsKey(DynamicPartitionProperty.ENABLE) ||
properties.containsKey(DynamicPartitionProperty.START_DAY_OF_WEEK) ||
properties.containsKey(DynamicPartitionProperty.START_DAY_OF_MONTH);
@@ -179,6 +192,7 @@ public class DynamicPartitionUtil {
String timeZone = properties.get(DynamicPartitionProperty.TIME_ZONE);
String end = properties.get(DynamicPartitionProperty.END);
String buckets = properties.get(DynamicPartitionProperty.BUCKETS);
+ String replicationNum = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
String enable = properties.get(DynamicPartitionProperty.ENABLE);
if (!((Strings.isNullOrEmpty(enable) &&
Strings.isNullOrEmpty(timeUnit) &&
@@ -284,6 +298,12 @@ public class DynamicPartitionUtil {
properties.remove(DynamicPartitionProperty.TIME_ZONE);
analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
}
+ if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
+ String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
+ checkReplicationNum(val);
+ properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
+ analyzedProperties.put(DynamicPartitionProperty.REPLICATION_NUM, val);
+ }
return analyzedProperties;
}
@@ -440,18 +460,6 @@ public class DynamicPartitionUtil {
return DateTimeFormatter.ofPattern(format).format(timeWithoutHourMinuteSecond);
}
- public static int estimateReplicateNum(OlapTable table) {
- int replicateNum = table.getDefaultReplicationNum();
- long maxPartitionId = 0;
- for (Partition partition: table.getPartitions()) {
- if (partition.getId() > maxPartitionId) {
- maxPartitionId = partition.getId();
- replicateNum = table.getPartitionInfo().getReplicationNum(partition.getId());
- }
- }
- return replicateNum;
- }
-
/**
* Used to indicate the start date.
* Taking the year as the granularity, it can indicate the month and day as the start date.
diff --git a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
index f7bf43b..30870fd 100644
--- a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1502,6 +1502,8 @@ public class ShowExecutor {
}
DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty();
String tableName = olapTable.getName();
+ int replicationNum = dynamicPartitionProperty.getReplicationNum();
+ replicationNum = (replicationNum == DynamicPartitionProperty.NOT_SET_REPLICATION_NUM) ? olapTable.getDefaultReplicationNum() : FeConstants.default_replication_num;
rows.add(Lists.newArrayList(
tableName,
String.valueOf(dynamicPartitionProperty.getEnable()),
@@ -1510,6 +1512,7 @@ public class ShowExecutor {
String.valueOf(dynamicPartitionProperty.getEnd()),
dynamicPartitionProperty.getPrefix(),
String.valueOf(dynamicPartitionProperty.getBuckets()),
+ String.valueOf(replicationNum),
dynamicPartitionProperty.getStartOfInfo(),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.LAST_UPDATE_TIME),
dynamicPartitionScheduler.getRuntimeInfo(tableName, DynamicPartitionScheduler.LAST_SCHEDULER_TIME),
diff --git a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index ecc82c3..8084ae4 100644
--- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -25,6 +25,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
@@ -93,6 +94,9 @@ public class DynamicPartitionTableTest {
"\"dynamic_partition.buckets\" = \"1\"\n" +
");";
createTable(createOlapTblStmt);
+ Database db = Catalog.getCurrentCatalog().getDb("default_cluster:test");
+ OlapTable table = (OlapTable) db.getTable("dynamic_partition_normal");
+ Assert.assertEquals(table.getTableProperty().getDynamicPartitionProperty().getReplicationNum(), DynamicPartitionProperty.NOT_SET_REPLICATION_NUM);
}
@Test
@@ -399,4 +403,39 @@ public class DynamicPartitionTableTest {
expectedException.expectMessage("errCode = 2, detailMessage = Unknown or incorrect time zone: 'invalid'");
createTable(createOlapTblStmt);
}
+
+ @Test
+ public void testSetDynamicPartitionReplicationNum() throws Exception {
+ String tableName = "dynamic_partition_replication_num";
+ String createOlapTblStmt = "CREATE TABLE test.`" + tableName + "` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\",\n" +
+ "\"dynamic_partition.buckets\" = \"1\",\n" +
+ "\"dynamic_partition.replication_num\" = \"2\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ Database db = Catalog.getCurrentCatalog().getDb("default_cluster:test");
+ OlapTable table = (OlapTable) db.getTable(tableName);
+ Assert.assertEquals(table.getTableProperty().getDynamicPartitionProperty().getReplicationNum(), 2);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org