You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/18 16:59:00 UTC
[doris] 06/18: [fix](dynamic partition) fix create hot partition failed without error response (#20996)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 42376bb2ed88931837ea3ca7cd725c63ccda751a
Author: yujun <yu...@gmail.com>
AuthorDate: Tue Jul 18 10:56:37 2023 +0800
[fix](dynamic partition) fix create hot partition failed without error response (#20996)
---
.../doris/common/util/DynamicPartitionUtil.java | 40 ++++++++++++++------
.../doris/catalog/DynamicPartitionTableTest.java | 44 +++++++++++++++++++++-
.../test_outer_join_with_window_function.groovy | 1 -
...t_outer_join_with_window_function_datev2.groovy | 1 -
.../test_dynamic_partition.groovy | 22 +++++++++++
5 files changed, 93 insertions(+), 15 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index 552ff2decb..7d0e5833fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -231,11 +231,24 @@ public class DynamicPartitionUtil {
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null);
}
- private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, Database db) throws DdlException {
+ private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int hotPartitionNum,
+ Database db) throws DdlException {
if (replicaAlloc.getTotalReplicaNum() <= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO);
}
+
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null);
+ if (hotPartitionNum <= 0) {
+ return;
+ }
+
+ try {
+ Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD);
+ } catch (DdlException e) {
+ throw new DdlException("Failed to find enough backend for ssd storage medium. When setting "
+ + DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the hot partitions will store "
+ + "in ssd. Please check the replication num,replication tag and storage medium.");
+ }
}
private static void checkHotPartitionNum(String val) throws DdlException {
@@ -600,28 +613,33 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
}
+ int hotPartitionNum = 0;
+ if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
+ String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
+ checkHotPartitionNum(val);
+ hotPartitionNum = Integer.parseInt(val);
+ properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
+ analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val);
+ }
+
// check replication_allocation first, then replciation_num
+ ReplicaAllocation replicaAlloc = null;
if (properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) {
- ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
- checkReplicaAllocation(replicaAlloc, db);
+ replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
properties.remove(DynamicPartitionProperty.REPLICATION_ALLOCATION);
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION, replicaAlloc.toCreateStmt());
} else if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
checkReplicationNum(val, db);
properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
+ replicaAlloc = new ReplicaAllocation(Short.valueOf(val));
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
- new ReplicaAllocation(Short.valueOf(val)).toCreateStmt());
+ replicaAlloc.toCreateStmt());
} else {
- checkReplicaAllocation(olapTable.getDefaultReplicaAllocation(), db);
+ replicaAlloc = olapTable.getDefaultReplicaAllocation();
}
+ checkReplicaAllocation(replicaAlloc, hotPartitionNum, db);
- if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
- String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
- checkHotPartitionNum(val);
- properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
- analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val);
- }
if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) {
String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS);
checkReservedHistoryPeriodValidate(reservedHistoryPeriods,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 3609e56592..22c19fb159 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;
@@ -82,6 +83,15 @@ public class DynamicPartitionTableTest {
UtFrameUtils.cleanDorisFeDir(runningDir);
}
+ private static void changeBeDisk(TStorageMedium storageMedium) {
+ List<Backend> backends = Env.getCurrentSystemInfo().getAllBackends();
+ for (Backend be : backends) {
+ for (DiskInfo diskInfo : be.getDisks().values()) {
+ diskInfo.setStorageMedium(storageMedium);
+ }
+ }
+ }
+
private static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Env.getCurrentEnv().createTable(createTableStmt);
@@ -970,6 +980,8 @@ public class DynamicPartitionTableTest {
@Test
public void testHotPartitionNum() throws Exception {
+ changeBeDisk(TStorageMedium.SSD);
+
Database testDb =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
// 1. hour
@@ -1188,9 +1200,11 @@ public class DynamicPartitionTableTest {
}
@Test(expected = DdlException.class)
- public void testHotPartitionNumAbnormal() throws Exception {
+ public void testHotPartitionNumAbnormalLT0() throws Exception {
+ changeBeDisk(TStorageMedium.SSD);
+
// dynamic_partition.hot_partition_num must larger than 0.
- String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1` (\n"
+ String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_lt0` (\n"
+ " `k1` datetime NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\"\n"
+ ") ENGINE=OLAP\n"
@@ -1211,6 +1225,32 @@ public class DynamicPartitionTableTest {
createTable(createOlapTblStmt);
}
+ @Test(expected = DdlException.class)
+ public void testHotPartitionNumAbnormalMissSSD() throws Exception {
+ changeBeDisk(TStorageMedium.HDD);
+
+ // when dynamic_partition.hot_partition_num > 0, it require ssd storage medium.
+ String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_miss_ssd` (\n"
+ + " `k1` datetime NULL COMMENT \"\",\n"
+ + " `k2` int NULL COMMENT \"\"\n"
+ + ") ENGINE=OLAP\n"
+ + "PARTITION BY RANGE(`k1`)\n"
+ + "()\n"
+ + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"dynamic_partition.enable\" = \"true\",\n"
+ + "\"dynamic_partition.start\" = \"-3\",\n"
+ + "\"dynamic_partition.end\" = \"3\",\n"
+ + "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+ + "\"dynamic_partition.time_unit\" = \"hour\",\n"
+ + "\"dynamic_partition.prefix\" = \"p\",\n"
+ + "\"dynamic_partition.buckets\" = \"1\",\n"
+ + "\"dynamic_partition.hot_partition_num\" = \"1\"\n"
+ + ");";
+ createTable(createOlapTblStmt);
+ }
+
@Test
public void testRuntimeInfo() throws Exception {
DynamicPartitionScheduler scheduler = new DynamicPartitionScheduler("test", 10);
diff --git a/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy b/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
index dfbea21562..3c195713ae 100644
--- a/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
+++ b/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
@@ -218,7 +218,6 @@ suite("test_outer_join_with_with_window_function") {
"dynamic_partition.buckets" = "4",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.history_partition_num" = "50",
- "dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.start_day_of_month" = "1",
"in_memory" = "false",
diff --git a/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy b/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
index 169e92c8ba..16abf0fb8a 100644
--- a/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
+++ b/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
@@ -222,7 +222,6 @@ suite("test_outer_join_with_window_function_datev2") {
"dynamic_partition.buckets" = "4",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.history_partition_num" = "50",
- "dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.start_day_of_month" = "1",
"in_memory" = "false",
diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
index 1a69dda4c7..f4680ff681 100644
--- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
+++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
@@ -149,4 +149,26 @@ suite("test_dynamic_partition") {
exception "errCode = 2,"
}
sql "drop table if exists dy_par_bad"
+ test {
+ sql """
+ CREATE TABLE IF NOT EXISTS dy_par_bad
+ ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
+ AGGREGATE KEY(k1,k2)
+ PARTITION BY RANGE(k1) ( )
+ DISTRIBUTED BY HASH(k1) BUCKETS 3
+ PROPERTIES (
+ "dynamic_partition.enable"="true",
+ "dynamic_partition.end"="3",
+ "dynamic_partition.start"="-3",
+ "dynamic_partition.prefix"="p",
+ "dynamic_partition.time_unit"="DAY",
+ "dynamic_partition.create_history_partition"="true",
+ "dynamic_partition.hot_partition_num" = "2",
+ "dynamic_partition.replication_allocation" = "tag.location.default: 1")
+ """
+ // check exception message contains
+ // dynamic_partition.hot_partition_num > 0 will require ssd storage medium
+ exception "errCode = 2,"
+ }
+ sql "drop table if exists dy_par_bad"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org