You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/02/19 14:50:12 UTC
[doris] 17/30: [improvement](dynamic-partition) add storage_medium property for dynamic partition (#16648)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9983feddcbb7b91effc25b3a7845593c6443b469
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Tue Feb 14 15:14:52 2023 +0800
[improvement](dynamic-partition) add storage_medium property for dynamic partition (#16648)
---
.../docs/advanced/partition/dynamic-partition.md | 7 +++++
.../docs/advanced/partition/dynamic-partition.md | 8 ++++++
.../doris/catalog/DynamicPartitionProperty.java | 11 +++++++-
.../doris/clone/DynamicPartitionScheduler.java | 32 ++++++++++++++++------
.../doris/common/util/DynamicPartitionUtil.java | 17 ++++++++++++
5 files changed, 66 insertions(+), 9 deletions(-)
diff --git a/docs/en/docs/advanced/partition/dynamic-partition.md b/docs/en/docs/advanced/partition/dynamic-partition.md
index ce6bc02ba8..92546f4d95 100644
--- a/docs/en/docs/advanced/partition/dynamic-partition.md
+++ b/docs/en/docs/advanced/partition/dynamic-partition.md
@@ -181,6 +181,13 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`:
Otherwise, every `[...,...]` in `reserved_history_periods` is a couple of properties, and they should be set at the same time. And the first date can't be larger than the second one.
+- `dynamic_partition.storage_medium`
+
+ <version since="dev"></version>
+
+ Specifies the default storage medium for the created dynamic partition. HDD is the default, SSD can be selected.
+
+ Note that when set to SSD, the `hot_partition_num` property will no longer take effect, all partitions will default to SSD storage media and the cooldown time will be 9999-12-31 23:59:59.
#### Create History Partition Rules
diff --git a/docs/zh-CN/docs/advanced/partition/dynamic-partition.md b/docs/zh-CN/docs/advanced/partition/dynamic-partition.md
index 5db9a504b7..1832544447 100644
--- a/docs/zh-CN/docs/advanced/partition/dynamic-partition.md
+++ b/docs/zh-CN/docs/advanced/partition/dynamic-partition.md
@@ -173,6 +173,14 @@ under the License.
这两个时间段的分区。其中，`reserved_history_periods` 的每一个 `[...,...]` 是一对设置项，两者需要同时被设置，且第一个时间不能大于第二个时间。
+- `dynamic_partition.storage_medium`
+
+ <version since="dev"></version>
+
+ 指定创建的动态分区的默认存储介质。默认是 HDD，可选择 SSD。
+
+ 注意，当设置为SSD时，`hot_partition_num` 属性将不再生效，所有分区将默认为 SSD 存储介质并且冷却时间为 9999-12-31 23:59:59。
+
#### 创建历史分区规则
当 `create_history_partition` 为 `true`，即开启创建历史分区功能时，Doris 会根据 `dynamic_partition.start` 和 `dynamic_partition.history_partition_num` 来决定创建历史分区的个数。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
index 88446394e0..8f5e9d359f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.DynamicPartitionUtil;
@@ -47,6 +48,7 @@ public class DynamicPartitionProperty {
public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num";
public static final String RESERVED_HISTORY_PERIODS = "dynamic_partition.reserved_history_periods";
public static final String STORAGE_POLICY = "dynamic_partition.storage_policy";
+ public static final String STORAGE_MEDIUM = "dynamic_partition.storage_medium";
public static final int MIN_START_OFFSET = Integer.MIN_VALUE;
public static final int MAX_END_OFFSET = Integer.MAX_VALUE;
@@ -74,6 +76,7 @@ public class DynamicPartitionProperty {
private int hotPartitionNum;
private String reservedHistoryPeriods;
private String storagePolicy;
+ private String storageMedium; // ssd or hdd
public DynamicPartitionProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
@@ -94,6 +97,7 @@ public class DynamicPartitionProperty {
this.reservedHistoryPeriods = properties.getOrDefault(
RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS);
this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, "");
+ this.storageMedium = properties.getOrDefault(STORAGE_MEDIUM, Config.default_storage_medium);
createStartOfs(properties);
} else {
this.exist = false;
@@ -177,6 +181,10 @@ public class DynamicPartitionProperty {
return storagePolicy;
}
+ public String getStorageMedium() {
+ return storageMedium;
+ }
+
public String getStartOfInfo() {
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return startOfWeek.toDisplayInfo();
@@ -220,7 +228,8 @@ public class DynamicPartitionProperty {
+ ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\""
+ ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\""
+ ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\""
- + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\"";
+ + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\""
+ + ",\n\"" + STORAGE_MEDIUM + "\" = \"" + storageMedium + "\"";
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\"";
} else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 3cbbe27c17..7c20455742 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -26,6 +26,7 @@ import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.RandomDistributionDesc;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.DynamicPartitionProperty;
@@ -298,10 +299,8 @@ public class DynamicPartitionScheduler extends MasterDaemon {
dynamicPartitionProperty.getReplicaAllocation().toCreateStmt());
}
- if (hotPartitionNum > 0) {
- // set storage_medium and storage_cooldown_time based on dynamic_partition.hot_partition_num
- setStorageMediumProperty(partitionProperties, dynamicPartitionProperty, now, hotPartitionNum, idx);
- }
+ // set storage_medium and storage_cooldown_time based on dynamic_partition.hot_partition_num
+ setStorageMediumProperty(partitionProperties, dynamicPartitionProperty, now, hotPartitionNum, idx);
if (StringUtils.isNotEmpty(storagePolicyName)) {
setStoragePolicyProperty(partitionProperties, dynamicPartitionProperty, now, idx, storagePolicyName);
@@ -309,7 +308,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
String partitionName = dynamicPartitionProperty.getPrefix()
+ DynamicPartitionUtil.getFormattedPartitionName(dynamicPartitionProperty.getTimeZone(),
- prevBorder, dynamicPartitionProperty.getTimeUnit());
+ prevBorder, dynamicPartitionProperty.getTimeUnit());
SinglePartitionDesc rangePartitionDesc = new SinglePartitionDesc(true, partitionName,
partitionKeyDesc, partitionProperties);
@@ -332,14 +331,31 @@ public class DynamicPartitionScheduler extends MasterDaemon {
return addPartitionClauses;
}
+ /**
+ * If dynamic_partition.storage_medium is set to SSD,
+ * ignore hot_partition_num property and set to (SSD, 9999-12-31 23:59:59)
+ * Else, if hot partition num is set, set storage medium to SSD due to time.
+ *
+ * @param partitionProperties
+ * @param property
+ * @param now
+ * @param hotPartitionNum
+ * @param offset
+ */
private void setStorageMediumProperty(HashMap<String, String> partitionProperties,
DynamicPartitionProperty property, ZonedDateTime now, int hotPartitionNum, int offset) {
- if (offset + hotPartitionNum <= 0) {
+ if ((hotPartitionNum <= 0 || offset + hotPartitionNum <= 0) && !property.getStorageMedium()
+ .equalsIgnoreCase("ssd")) {
return;
}
+ String cooldownTime;
+ if (property.getStorageMedium().equalsIgnoreCase("ssd")) {
+ cooldownTime = TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS);
+ } else {
+ cooldownTime = DynamicPartitionUtil.getPartitionRangeString(
+ property, now, offset + hotPartitionNum, DynamicPartitionUtil.DATETIME_FORMAT);
+ }
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, TStorageMedium.SSD.name());
- String cooldownTime = DynamicPartitionUtil.getPartitionRangeString(
- property, now, offset + hotPartitionNum, DynamicPartitionUtil.DATETIME_FORMAT);
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME, cooldownTime);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index 60247b4ec7..b51aeaf9bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -39,6 +39,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
import org.apache.doris.policy.StoragePolicy;
+import org.apache.doris.thrift.TStorageMedium;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -365,6 +366,14 @@ public class DynamicPartitionUtil {
}
}
+ private static void checkStorageMedium(String storageMedium) throws DdlException {
+ try {
+ TStorageMedium.valueOf(storageMedium.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new DdlException("invalid storage medium: " + storageMedium + ". Should be SSD or HDD");
+ }
+ }
+
private static SimpleDateFormat getSimpleDateFormat(String timeUnit) {
if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return new SimpleDateFormat(DATETIME_FORMAT);
@@ -624,6 +633,14 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.STORAGE_POLICY, remoteStoragePolicy);
}
}
+ if (properties.containsKey(DynamicPartitionProperty.STORAGE_MEDIUM)) {
+ String storageMedium = properties.get(DynamicPartitionProperty.STORAGE_MEDIUM);
+ checkStorageMedium(storageMedium);
+ properties.remove(DynamicPartitionProperty.STORAGE_MEDIUM);
+ if (!Strings.isNullOrEmpty(storageMedium)) {
+ analyzedProperties.put(DynamicPartitionProperty.STORAGE_MEDIUM, storageMedium);
+ }
+ }
return analyzedProperties;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org