You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/06/13 08:27:19 UTC
[incubator-doris] branch master updated: [Dynamic Partition] Use
ZonedDateTime to support set timezone (#3799)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 414a0a3 [Dynamic Partition] Use ZonedDateTime to support set timezone (#3799)
414a0a3 is described below
commit 414a0a35e53648a260c1d6eee70c9c2557da2425
Author: WingC <10...@qq.com>
AuthorDate: Sat Jun 13 03:27:09 2020 -0500
[Dynamic Partition] Use ZonedDateTime to support set timezone (#3799)
This CL mainly support timezone in dynamic partition:
1. use new Java Time API to replace Calendar.
2. support set time zone in dynamic partition parameters.
---
docs/en/administrator-guide/dynamic-partition.md | 6 +-
.../zh-CN/administrator-guide/dynamic-partition.md | 6 +-
.../doris/catalog/DynamicPartitionProperty.java | 8 +-
.../doris/clone/DynamicPartitionScheduler.java | 18 ++--
.../doris/common/util/DynamicPartitionUtil.java | 70 ++++++++--------
.../org/apache/doris/common/util/TimeUtils.java | 7 ++
.../doris/catalog/DynamicPartitionTableTest.java | 96 +++++++++++++++++++++-
.../common/util/DynamicPartitionUtilTest.java | 55 +++++++------
8 files changed, 191 insertions(+), 75 deletions(-)
diff --git a/docs/en/administrator-guide/dynamic-partition.md b/docs/en/administrator-guide/dynamic-partition.md
index 26dd3a8..f4c6a54 100644
--- a/docs/en/administrator-guide/dynamic-partition.md
+++ b/docs/en/administrator-guide/dynamic-partition.md
@@ -89,6 +89,10 @@ The rules of dynamic partition are prefixed with `dynamic_partition.`:
When specified as `MONTH`, the suffix format of the dynamically created partition name is `yyyyMM`, for example, `202003`.
+* `dynamic_partition.time_zone`
+
+ The time zone of the dynamic partition, if not filled in, defaults to the time zone of the current machine's system, such as `Asia/Shanghai`, if you want to know the supported TimeZone, you can found in `https://en.wikipedia.org/wiki/List_of_tz_database_time_zones`.
+
* `dynamic_partition.start`
The starting offset of the dynamic partition, usually a negative number. Depending on the `time_unit` attribute, based on the current day (week / month), the partitions with a partition range before this offset will be deleted. If not filled, the default is `-2147483648`, that is, the history partition will not be deleted.
@@ -306,4 +310,4 @@ mysql> SHOW DYNAMIC PARTITION TABLES;
HTTP protocal：
`curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000`
-
\ No newline at end of file
+
diff --git a/docs/zh-CN/administrator-guide/dynamic-partition.md b/docs/zh-CN/administrator-guide/dynamic-partition.md
index 3897742..709c154 100644
--- a/docs/zh-CN/administrator-guide/dynamic-partition.md
+++ b/docs/zh-CN/administrator-guide/dynamic-partition.md
@@ -87,6 +87,10 @@ under the License.
当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
+* `dynamic_partition.time_zone`
+
+ 动态分区的时区，如果不填写，则默认为当前机器的系统的时区，例如 `Asia/Shanghai`，如果想获取当前支持的时区设置，可以参考 `https://en.wikipedia.org/wiki/List_of_tz_database_time_zones`。
+
* `dynamic_partition.start`
动态分区的起始偏移，为负数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，分区范围在此偏移之前的分区将会被删除。如果不填写，则默认为 `-2147483648`，即不删除历史分区。
@@ -303,4 +307,4 @@ mysql> SHOW DYNAMIC PARTITION TABLES;
HTTP 协议：
- `curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000`
\ No newline at end of file
+ `curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000`
diff --git a/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
index 4bdd2fa..0eca48c 100644
--- a/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
+++ b/fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
@@ -24,7 +24,7 @@ import org.apache.doris.common.util.TimeUtils;
import java.util.Map;
import java.util.TimeZone;
-public class DynamicPartitionProperty{
+public class DynamicPartitionProperty {
public static final String TIME_UNIT = "dynamic_partition.time_unit";
public static final String START = "dynamic_partition.start";
public static final String END = "dynamic_partition.end";
@@ -33,6 +33,7 @@ public class DynamicPartitionProperty{
public static final String ENABLE = "dynamic_partition.enable";
public static final String START_DAY_OF_WEEK = "dynamic_partition.start_day_of_week";
public static final String START_DAY_OF_MONTH = "dynamic_partition.start_day_of_month";
+ public static final String TIME_ZONE = "dynamic_partition.time_zone";
public static final int MIN_START_OFFSET = Integer.MIN_VALUE;
@@ -46,15 +47,14 @@ public class DynamicPartitionProperty{
private int buckets;
private StartOfDate startOfWeek;
private StartOfDate startOfMonth;
- // TODO: support setting timezone.
private TimeZone tz = TimeUtils.getDefaultTimeZone();
-
public DynamicPartitionProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
this.exist = true;
this.enable = Boolean.parseBoolean(properties.get(ENABLE));
this.timeUnit = properties.get(TIME_UNIT);
+ this.tz = TimeUtils.getOrSystemTimeZone(properties.get(TIME_ZONE));
// In order to compatible dynamic add partition version
this.start = Integer.parseInt(properties.getOrDefault(START, String.valueOf(MIN_START_OFFSET)));
this.end = Integer.parseInt(properties.get(END));
@@ -132,11 +132,11 @@ public class DynamicPartitionProperty{
return tz;
}
-
@Override
public String toString() {
String res = ",\n\"" + ENABLE + "\" = \"" + enable + "\"" +
",\n\"" + TIME_UNIT + "\" = \"" + timeUnit + "\"" +
+ ",\n\"" + TIME_ZONE + "\" = \"" + tz + "\"" +
",\n\"" + START + "\" = \"" + start + "\"" +
",\n\"" + END + "\" = \"" + end + "\"" +
",\n\"" + PREFIX + "\" = \"" + prefix + "\"" +
diff --git a/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 999375b..d10f543 100644
--- a/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ b/fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -49,8 +49,8 @@ import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.time.ZonedDateTime;
import java.util.ArrayList;
-import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -131,12 +131,10 @@ public class DynamicPartitionScheduler extends MasterDaemon {
ArrayList<AddPartitionClause> addPartitionClauses = new ArrayList<>();
DynamicPartitionProperty dynamicPartitionProperty = olapTable.getTableProperty().getDynamicPartitionProperty();
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo();
- Calendar currentDate = Calendar.getInstance(dynamicPartitionProperty.getTimeZone());
+ ZonedDateTime now = ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
for (int i = 0; i <= dynamicPartitionProperty.getEnd(); i++) {
- String prevBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
- (Calendar) currentDate.clone(), i, partitionFormat);
- String nextBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
- (Calendar) currentDate.clone(), i + 1, partitionFormat);
+ String prevBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, i, partitionFormat);
+ String nextBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty, now, i + 1, partitionFormat);
PartitionValue lowerValue = new PartitionValue(prevBorder);
PartitionValue upperValue = new PartitionValue(nextBorder);
@@ -193,7 +191,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
return addPartitionClauses;
}
- /*
+ /**
* 1. get the range of [start, 0) as a reserved range.
* 2. get DropPartitionClause of partitions which range are before this reserved range.
*/
@@ -205,11 +203,11 @@ public class DynamicPartitionScheduler extends MasterDaemon {
return dropPartitionClauses;
}
- Calendar currentDate = Calendar.getInstance(dynamicPartitionProperty.getTimeZone());
+ ZonedDateTime now = ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
String lowerBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
- (Calendar) currentDate.clone(), dynamicPartitionProperty.getStart(), partitionFormat);
+ now, dynamicPartitionProperty.getStart(), partitionFormat);
String upperBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
- (Calendar) currentDate.clone(), 0, partitionFormat);
+ now, 0, partitionFormat);
PartitionValue lowerPartitionValue = new PartitionValue(lowerBorder);
PartitionValue upperPartitionValue = new PartitionValue(upperBorder);
Range<PartitionKey> reservePartitionKeyRange;
diff --git a/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index c39a33c..0ba6544 100644
--- a/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -46,6 +46,9 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.DayOfWeek;
import java.time.Month;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
@@ -153,6 +156,7 @@ public class DynamicPartitionUtil {
return false;
}
return properties.containsKey(DynamicPartitionProperty.TIME_UNIT) ||
+ properties.containsKey(DynamicPartitionProperty.TIME_ZONE) ||
properties.containsKey(DynamicPartitionProperty.START) ||
properties.containsKey(DynamicPartitionProperty.END) ||
properties.containsKey(DynamicPartitionProperty.PREFIX) ||
@@ -172,11 +176,13 @@ public class DynamicPartitionUtil {
String timeUnit = properties.get(DynamicPartitionProperty.TIME_UNIT);
String prefix = properties.get(DynamicPartitionProperty.PREFIX);
String start = properties.get(DynamicPartitionProperty.START);
+ String timeZone = properties.get(DynamicPartitionProperty.TIME_ZONE);
String end = properties.get(DynamicPartitionProperty.END);
String buckets = properties.get(DynamicPartitionProperty.BUCKETS);
String enable = properties.get(DynamicPartitionProperty.ENABLE);
if (!((Strings.isNullOrEmpty(enable) &&
Strings.isNullOrEmpty(timeUnit) &&
+ Strings.isNullOrEmpty(timeZone) &&
Strings.isNullOrEmpty(prefix) &&
Strings.isNullOrEmpty(start) &&
Strings.isNullOrEmpty(end) &&
@@ -199,6 +205,9 @@ public class DynamicPartitionUtil {
if (Strings.isNullOrEmpty(buckets)) {
throw new DdlException("Must assign dynamic_partition.buckets properties");
}
+ if (Strings.isNullOrEmpty(timeZone)) {
+ properties.put(DynamicPartitionProperty.TIME_ZONE, ZoneId.systemDefault().toString());
+ }
}
return true;
}
@@ -268,6 +277,13 @@ public class DynamicPartitionUtil {
properties.remove(DynamicPartitionProperty.START_DAY_OF_WEEK);
analyzedProperties.put(DynamicPartitionProperty.START_DAY_OF_WEEK, val);
}
+
+ if (properties.containsKey(DynamicPartitionProperty.TIME_ZONE)) {
+ String val = properties.get(DynamicPartitionProperty.TIME_ZONE);
+ TimeUtils.checkTimeZoneValidAndStandardize(val);
+ properties.remove(DynamicPartitionProperty.TIME_ZONE);
+ analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
+ }
return analyzedProperties;
}
@@ -350,20 +366,19 @@ public class DynamicPartitionUtil {
// return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
// TODO: support HOUR and YEAR
- public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ public static String getPartitionRangeString(DynamicPartitionProperty property, ZonedDateTime current,
int offset, String format) {
String timeUnit = property.getTimeUnit();
- TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return getPartitionRangeOfDay(current, offset, tz, format);
+ return getPartitionRangeOfDay(current, offset, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), format);
} else { // MONTH
- return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), format);
}
}
- /*
+ /**
* return formatted string of partition range in DAY granularity.
* offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
* format: the format of the return date string.
@@ -372,12 +387,11 @@ public class DynamicPartitionUtil {
* Today is 2020-05-24, offset = -1
* It will return 2020-05-23
*/
- private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
- current.add(Calendar.DATE, offset);
- return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ private static String getPartitionRangeOfDay(ZonedDateTime current, int offset, String format) {
+ return getFormattedTimeWithoutHourMinuteSecond(current.plusDays(offset), format);
}
- /*
+ /**
* return formatted string of partition range in WEEK granularity.
* offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
* startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
@@ -387,20 +401,17 @@ public class DynamicPartitionUtil {
* Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
* It will return 2020-05-20 (Wednesday of last week)
*/
- private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
- String format) {
+ private static String getPartitionRangeOfWeek(ZonedDateTime current, int offset, StartOfDate startOf, String format) {
Preconditions.checkArgument(startOf.isStartOfWeek());
// 1. get the offset week
- current.add(Calendar.WEEK_OF_YEAR, offset);
+ ZonedDateTime offsetWeek = current.plusWeeks(offset);
// 2. get the date of `startOf` week
- int day = current.get(Calendar.DAY_OF_WEEK);
- // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
- day = (day == 1 ? 7 : day - 1);
- current.add(Calendar.DATE, (startOf.dayOfWeek - day));
- return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ int day = offsetWeek.getDayOfWeek().getValue();
+ ZonedDateTime resultTime = offsetWeek.plusDays(startOf.dayOfWeek - day);
+ return getFormattedTimeWithoutHourMinuteSecond(resultTime, format);
}
- /*
+ /**
* return formatted string of partition range in MONTH granularity.
* offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month.
* startOf: Define the start date of each month. 1 means start on the 1st of every month.
@@ -410,28 +421,23 @@ public class DynamicPartitionUtil {
* Today is 2020-05-24, offset = 1, startOf.month = 3
* It will return 2020-06-03
*/
- private static String getPartitionRangeOfMonth(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
- String format) {
+ private static String getPartitionRangeOfMonth(ZonedDateTime current, int offset, StartOfDate startOf, String format) {
Preconditions.checkArgument(startOf.isStartOfMonth());
// 1. Get the offset date.
int realOffset = offset;
- int currentDay = current.get(Calendar.DATE);
+ int currentDay = current.getDayOfMonth();
if (currentDay < startOf.day) {
// eg: today is 2020-05-20, `startOf.day` is 25, and offset is 0.
// we should return 2020-04-25, which is the last month.
realOffset -= 1;
}
- current.add(Calendar.MONTH, realOffset);
- current.set(Calendar.DATE, startOf.day);
- return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ ZonedDateTime resultTime = current.plusMonths(realOffset).withDayOfMonth(startOf.day);
+ return getFormattedTimeWithoutHourMinuteSecond(resultTime, format);
}
- private static String getFormattedTimeWithoutHourMinuteSecond(Calendar calendar, String format) {
- calendar.set(Calendar.HOUR_OF_DAY, 0);
- calendar.set(Calendar.MINUTE, 0);
- calendar.set(Calendar.SECOND, 0);
- SimpleDateFormat dateFormat = new SimpleDateFormat(format);
- return dateFormat.format(calendar.getTime());
+ private static String getFormattedTimeWithoutHourMinuteSecond(ZonedDateTime zonedDateTime, String format) {
+ ZonedDateTime timeWithoutHourMinuteSecond = zonedDateTime.withHour(0).withMinute(0).withSecond(0);
+ return DateTimeFormatter.ofPattern(format).format(timeWithoutHourMinuteSecond);
}
public static int estimateReplicateNum(OlapTable table) {
@@ -446,7 +452,7 @@ public class DynamicPartitionUtil {
return replicateNum;
}
- /*
+ /**
* Used to indicate the start date.
* Taking the year as the granularity, it can indicate the month and day as the start date.
* Taking the month as the granularity, it can indicate the date of as the start date.
diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
index f8c0c1d..8678458 100644
--- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
+++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -129,6 +129,13 @@ public class TimeUtils {
return TimeZone.getTimeZone(ZoneId.of(DEFAULT_TIME_ZONE, timeZoneAliasMap));
}
+ public static TimeZone getOrSystemTimeZone(String timeZone) {
+ if (timeZone == null) {
+ timeZone = ZoneId.systemDefault().toString();
+ }
+ return TimeZone.getTimeZone(timeZone);
+ }
+
public static String longToTimeString(long timeStamp, SimpleDateFormat dateFormat) {
if (timeStamp <= 0L) {
return "N/A";
diff --git a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index b744cf2..ecc82c3 100644
--- a/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++ b/fe/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -186,7 +186,6 @@ public class DynamicPartitionTableTest {
createTable(createOlapTblStmt);
}
-
@Test
public void testMissEnd() throws Exception {
String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_end` (\n" +
@@ -218,7 +217,6 @@ public class DynamicPartitionTableTest {
createTable(createOlapTblStmt);
}
-
@Test
public void testMissBuckets() throws Exception {
String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_buckets` (\n" +
@@ -307,4 +305,98 @@ public class DynamicPartitionTableTest {
expectedException.expectMessage("errCode = 2, detailMessage = Dynamic partition only support single-column range partition");
createTable(createOlapTblStmt);
}
+
+ @Test
+ public void testMissTimeZone() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_miss_time_zone` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.buckets\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ }
+
+ @Test
+ public void testNormalTimeZone() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_time_zone` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.buckets\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.time_zone\" = \"Asia/Shanghai\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\"\n" +
+ ");";
+ createTable(createOlapTblStmt);
+ }
+
+ @Test
+ public void testInvalidTimeZone() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_invalid_time_zone` (\n" +
+ " `k1` date NULL COMMENT \"\",\n" +
+ " `k2` int NULL COMMENT \"\",\n" +
+ " `k3` smallint NULL COMMENT \"\",\n" +
+ " `v1` varchar(2048) NULL COMMENT \"\",\n" +
+ " `v2` datetime NULL COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "PARTITION BY RANGE (k1)\n" +
+ "(\n" +
+ "PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" +
+ "PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" +
+ "PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" +
+ ")\n" +
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"dynamic_partition.enable\" = \"true\",\n" +
+ "\"dynamic_partition.start\" = \"-3\",\n" +
+ "\"dynamic_partition.end\" = \"3\",\n" +
+ "\"dynamic_partition.buckets\" = \"3\",\n" +
+ "\"dynamic_partition.time_unit\" = \"day\",\n" +
+ "\"dynamic_partition.time_zone\" = \"invalid\",\n" +
+ "\"dynamic_partition.prefix\" = \"p\"\n" +
+ ");";
+ expectedException.expect(DdlException.class);
+ expectedException.expectMessage("errCode = 2, detailMessage = Unknown or incorrect time zone: 'invalid'");
+ createTable(createOlapTblStmt);
+ }
}
diff --git a/fe/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java b/fe/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java
index 3339c15..8143c18 100644
--- a/fe/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java
+++ b/fe/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java
@@ -24,8 +24,16 @@ import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Test;
+import java.sql.Time;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.time.DateTimeException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
@@ -53,39 +61,36 @@ public class DynamicPartitionUtilTest {
return prop;
}
- private static Calendar getCalendarWithDate(String dateStr) throws ParseException {
- SimpleDateFormat sdf = new SimpleDateFormat(FORMAT);
- Calendar calendar = Calendar.getInstance(TimeUtils.getDefaultTimeZone());
- Date date = sdf.parse(dateStr);
- calendar.setTime(date);
- return calendar;
+ private static ZonedDateTime getZonedDateTimeFromStr(String dateStr) throws DateTimeException {
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(FORMAT);
+ return LocalDate.parse(dateStr, formatter).atStartOfDay(ZoneId.systemDefault());
}
@Test
- public void testGetPartitionRangeString() throws ParseException {
+ public void testGetPartitionRangeString() throws DateTimeException {
// TimeUnit: DAY
// 1. 2020-05-25, offset -7
DynamicPartitionProperty property = new DynamicPartitionProperty(getDynamProp("DAY", -3, 3, -1, -1));
- String res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), -7,
+ String res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), -7,
FORMAT);
Assert.assertEquals("2020-05-18", res);
String partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
Assert.assertEquals("20200518", partName);
// 2. 2020-05-25, offset 0
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-25", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
Assert.assertEquals("20200525", partName);
// 3. 2020-05-25, offset 7
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), 7,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 7,
FORMAT);
Assert.assertEquals("2020-06-01", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
Assert.assertEquals("20200601", partName);
// 4. 2020-02-28, offset 3
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-02-28"), 3,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-28"), 3,
FORMAT);
Assert.assertEquals("2020-03-02", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
@@ -94,7 +99,7 @@ public class DynamicPartitionUtilTest {
// TimeUnit: WEEK
// 1. 2020-05-25, start day: MONDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-25", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -102,7 +107,7 @@ public class DynamicPartitionUtilTest {
// 2. 2020-05-28, start day: MONDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-28"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-28"), 0,
FORMAT);
Assert.assertEquals("2020-05-25", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -110,7 +115,7 @@ public class DynamicPartitionUtilTest {
// 3. 2020-05-25, start day: SUNDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 7, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-31", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -118,7 +123,7 @@ public class DynamicPartitionUtilTest {
// 4. 2020-05-25, start day: MONDAY, offset -2
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), -2,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), -2,
FORMAT);
Assert.assertEquals("2020-05-11", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -126,7 +131,7 @@ public class DynamicPartitionUtilTest {
// 5. 2020-02-29, start day: WED, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 3, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-02-29"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 0,
FORMAT);
Assert.assertEquals("2020-02-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -134,7 +139,7 @@ public class DynamicPartitionUtilTest {
// 6. 2020-02-29, start day: TUS, offset 1
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 2, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-02-29"), 1,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 1,
FORMAT);
Assert.assertEquals("2020-03-03", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -142,7 +147,7 @@ public class DynamicPartitionUtilTest {
// 6. 2020-01-01, start day: MONDAY, offset -1
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-01-01"), -1,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-01-01"), -1,
FORMAT);
Assert.assertEquals("2019-12-23", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -150,7 +155,7 @@ public class DynamicPartitionUtilTest {
// 6. 2020-01-01, start day: MONDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-01-01"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-01-01"), 0,
FORMAT);
Assert.assertEquals("2019-12-30", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
@@ -159,7 +164,7 @@ public class DynamicPartitionUtilTest {
// TimeUnit: MONTH
// 1. 2020-05-25, start day: 1, offset 0
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 1));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-01", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
@@ -167,7 +172,7 @@ public class DynamicPartitionUtilTest {
// 2. 2020-05-25, start day: 26, offset 0
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 26));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-04-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
@@ -175,7 +180,7 @@ public class DynamicPartitionUtilTest {
// 3. 2020-05-25, start day: 26, offset -1
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 26));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-05-25"), -1,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), -1,
FORMAT);
Assert.assertEquals("2020-03-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
@@ -183,7 +188,7 @@ public class DynamicPartitionUtilTest {
// 4. 2020-02-29, start day: 26, offset 3
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 26));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-02-29"), 3,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 3,
FORMAT);
Assert.assertEquals("2020-05-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
@@ -191,7 +196,7 @@ public class DynamicPartitionUtilTest {
// 5. 2020-02-29, start day: 27, offset 0
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 27));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-02-29"), 0,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 0,
FORMAT);
Assert.assertEquals("2020-02-27", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
@@ -199,7 +204,7 @@ public class DynamicPartitionUtilTest {
// 6. 2020-02-29, start day: 27, offset -3
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 27));
- res = DynamicPartitionUtil.getPartitionRangeString(property, getCalendarWithDate("2020-02-29"), -3,
+ res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), -3,
FORMAT);
Assert.assertEquals("2019-11-27", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org