You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2020/05/25 06:51:29 UTC
[GitHub] [incubator-doris] morningman opened a new pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
morningman opened a new pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679
Problem is described in ISSUE #3678
This CL mainly changed to rule of creating dynamic partition.
1. If time unit is DAY, the logic remains unchanged.
2. If time unit is WEEK, the logical changes are as follows:
1. Allow to set the start day of every week, the default is Monday. Optional Monday to Sunday
2. Assuming that the starting day is a Tuesday, the range of the partition is Tuesday of the week to Monday of the next week.
3. If time unit is MONTH, the logical changes are as follows:
1. Allow to set the start date of each month. The default is 1st, and can be selected from 1st to 28th.
2. Assuming that the starting date is the 2nd, the range of the partition is from the 2nd of this month to the 1st of the next month.
4. The `SHOW DYNAMIC PARTITION TABLES` statement adds a `StartOf` column to show the start day of week or month.
It is recommended to refer to the example in `dynamic-partition.md` to understand.
TODO:
Better to support HOUR and YEAR time unit. Maybe in next PR.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429859638
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
+ current.add(Calendar.DATE, (startOf.dayOfWeek - day));
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in MONTH granularity.
+ * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month.
+ * startOf: Define the start date of each month. 1 means start on the 1st of every month.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = 1, startOf.month = 3
+ * It will return 2020-06-03
+ */
+ private static String getPartitionRangeOfMonth(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
Review comment:
Maybe in next PR? This PR is already too big
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429859199
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -114,6 +118,34 @@ private static void checkEnable(String enable) throws DdlException {
}
}
+ private static void checkStartDayOfMonth(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
+ }
+ try {
+ int dayOfMonth = Integer.parseInt(val);
+ if (dayOfMonth < 1 || dayOfMonth > 28) {
+ throw new DdlException(DynamicPartitionProperty.START_DAY_OF_MONTH + " should between 1 and 28");
+ }
+ } catch (NumberFormatException e) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
+ }
+ }
+
+ private static void checkStartDayOfWeek(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_WEEK);
+ }
+ try {
+ int dayOfMonth = Integer.parseInt(val);
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wutiangan commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
wutiangan commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429809541
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
Review comment:
you‘d better add Parentheses
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429859708
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429854433
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
Review comment:
java doc comment is start with `/** xx */` , maybe we can unify it?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wutiangan commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
wutiangan commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429804756
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -161,48 +281,26 @@ SHOW DYNAMIC PARTITION TABLES;
* dynamic\_partition\_enable
- 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。
+ 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。可以通过修改 fe.conf 中的参数并重启 FE 生效。也可以在运行时执行以下命令生效：
-* dynamic\_partition\_check\_interval\_seconds
-
- 动态分区线程的执行频率，默认为3600(1个小时)，即每1个小时进行一次调度
+ MySQL 协议：
-### HTTP Restful API
-
-Doris 提供了修改动态分区配置参数的 HTTP Restful API，用于运行时修改动态分区配置参数。
-
-该 API 实现在 FE 端，使用 `fe_host:fe_http_port` 进行访问。需要 ADMIN 权限。
-
-1. 将 dynamic_partition_enable 设置为 true 或 false
+ `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`
+
+ HTTP 协议：
+
+ `curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true`
- * 标记为 true
+ 若要全局关闭动态分区，则设置此参数为 false 即可。
- ```
- GET /api/_set_config?dynamic_partition_enable=true
-
- 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true
-
- 返回：200
- ```
-
- * 标记为 false
+* dynamic\_partition\_check\_interval\_seconds
+
+ 动态分区线程的执行频率，默认为3600(1个小时)，即每1个小时进行一次调度。可以通过修改 fe.conf 中的参数并重启 FE 生效。也可以在运行时执行以下命令修改：
- ```
- GET /api/_set_config?dynamic_partition_enable=false
-
- 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false
-
- 返回：200
- ```
+ MySQL 协议：
+
+ `ADMIN SET FRONTEND CONFIG ("dynamic_partition_check_interval_seconds" = "7200")`
-2. 设置 dynamic partition 的调度频率
+ HTTP 协议：
- * 设置调度时间为12小时调度一次
-
- ```
- GET /api/_set_config?dynamic_partition_check_interval_seconds=432000
-
- 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000
-
- 返回：200
- ```
+ `curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000`
Review comment:
设置http协议为啥用GET方法而不是SET方法？
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wutiangan commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
wutiangan commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429803209
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -41,117 +39,239 @@ under the License.
在某些使用场景下，用户会将表按照天进行分区划分，每天定时执行例行任务，这时需要使用方手动管理分区，否则可能由于使用方没有创建分区导致数据导入失败，这给使用方带来了额外的维护成本。
-在实现方式上, FE会启动一个后台线程，根据fe.conf中`dynamic_partition_enable` 及 `dynamic_partition_check_interval_seconds`参数决定该线程是否启动以及该线程的调度频率。每次调度时，会在注册表中读取动态分区表的属性，并根据动态分区属性动态添加及删除分区。
-
-建表时，在properties中指定dynamic_partition属性，FE首先对动态分区属性进行解析，校验输入参数的合法性，然后将对应的属性持久化到FE的元数据中，并将该表注册到动态分区列表中，后台线程会根据配置参数定期对动态分区列表进行扫描，读取表的动态分区属性，执行添加分区及删除分区的任务，每次的调度信息会保留在FE的内存中（重启后则丢失），可以通过`SHOW DYNAMIC PARTITION TABLES`查看调度任务是否成功，如果存在分区创建或删除失败，会将失败信息输出。
+通过动态分区功能，用户可以在建表时设定动态分区的规则。FE 会启动一个后台线程，根据用户指定的规则创建或删除分区。用户也可以在运行时对现有规则进行变更。
## 使用方式
-### 动态分区属性参数说明:
+动态分区的规则可以在建表时指定，或者在运行时进行修改。当前仅支持对单分区列的分区表设定动态分区规则。
+
+* 建表时指定：
+
+ ```
+ CREATE TABLE tbl1
+ (...)
+ PROPERTIES
+ (
+ "dynamic_partition.prop1" = "value1",
+ "dynamic_partition.prop2" = "value2",
+ ...
+ )
+ ```
-`dynamic_partition.enable`: 是否开启动态分区特性，可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。
+* 运行时修改
+ ```
+ ALTER TABLE tbl1 SET
+ (
+ "dynamic_partition.prop1" = "value1",
+ "dynamic_partition.prop2" = "value2",
+ ...
+ )
+ ```
-`dynamic_partition.time_unit`: 动态分区调度的单位，可指定为 `DAY` `WEEK` `MONTH`，当指定为 `DAY` 时，动态创建的分区名后缀格式为`yyyyMMdd`，例如`20200325`。当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
+### 动态分区规则参数
-`dynamic_partition.start`: 动态分区的开始时间, 以当天为基准，超过该时间范围的分区将会被删除。如果不填写，则默认为`Integer.MIN_VALUE` 即 `-2147483648`。
+动态分区的规则参数都以 `dynamic_partition.` 为前缀：
+* `dynamic_partition.enable`
-`dynamic_partition.end`: 动态分区的结束时间, 以当天为基准，会提前创建N个单位的分区范围。
+ 是否开启动态分区特性。可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。如果为 `FALSE`，则 Doris 会忽略该表的动态分区规则。
-`dynamic_partition.prefix`: 动态创建的分区名前缀。
+* `dynamic_partition.time_unit`
-`dynamic_partition.buckets`: 动态创建的分区所对应的分桶数量。
+ 动态分区调度的单位。可指定为 `DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。
-### 建表
+ 当指定为 `DAY` 时，动态创建的分区名后缀格式为 `yyyyMMdd`，例如`20200325`。
+
+ 当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`。即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。
+
+ 当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
-建表时，可以在 `PROPERTIES` 中指定以下`dynamic_partition`属性，表示这个表是一个动态分区表。
+* `dynamic_partition.start`
-示例：
+ 动态分区的起始偏移，为负数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，分区范围在此偏移之前的分区将会被删除。如果不填写，则默认为 `-2147483648`，即不删除历史分区。
+
+* `dynamic_partition.end`
-```
-CREATE TABLE example_db.dynamic_partition
-(
-k1 DATE,
-k2 INT,
-k3 SMALLINT,
-v1 VARCHAR(2048),
-v2 DATETIME DEFAULT "2014-02-04 15:36:00"
-)
-ENGINE=olap
-DUPLICATE KEY(k1, k2, k3)
-PARTITION BY RANGE (k1)
-(
-PARTITION p20200321 VALUES LESS THAN ("2020-03-22"),
-PARTITION p20200322 VALUES LESS THAN ("2020-03-23"),
-PARTITION p20200323 VALUES LESS THAN ("2020-03-24"),
-PARTITION p20200324 VALUES LESS THAN ("2020-03-25")
-)
-DISTRIBUTED BY HASH(k2) BUCKETS 32
-PROPERTIES(
-"storage_medium" = "SSD",
-"dynamic_partition.enable" = "true",
-"dynamic_partition.time_unit" = "DAY",
-"dynamic_partition.start" = "-3",
-"dynamic_partition.end" = "3",
-"dynamic_partition.prefix" = "p",
-"dynamic_partition.buckets" = "32"
- );
-```
-创建一张动态分区表，指定开启动态分区特性，以当天为2020-03-25为例，在每次调度时，会删除分区上界小于 `2020-03-22` 的分区，为了避免删除非动态创建的分区，动态删除分区只会删除分区名符合动态创建分区规则的分区，例如分区名为a1, 则即使分区范围在待删除的分区范围内，也不会被删除。同时在调度时会提前创建今天以及以后3天（总共4天）的分区(若分区已存在则会忽略)，分区名根据指定前缀分别为`p20200325` `p20200326` `p20200327` `p20200328`,每个分区的分桶数量为32。同时会删除 `p20200321` 的分区，最终的分区范围如下:
-```
-[types: [DATE]; keys: [2020-03-22]; ‥types: [DATE]; keys: [2020-03-23]; )
-[types: [DATE]; keys: [2020-03-23]; ‥types: [DATE]; keys: [2020-03-24]; )
-[types: [DATE]; keys: [2020-03-24]; ‥types: [DATE]; keys: [2020-03-25]; )
-[types: [DATE]; keys: [2020-03-25]; ‥types: [DATE]; keys: [2020-03-26]; )
-[types: [DATE]; keys: [2020-03-26]; ‥types: [DATE]; keys: [2020-03-27]; )
-[types: [DATE]; keys: [2020-03-27]; ‥types: [DATE]; keys: [2020-03-28]; )
-[types: [DATE]; keys: [2020-03-28]; ‥types: [DATE]; keys: [2020-03-29]; )
-```
+ 动态分区的结束偏移，为正数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，提前创建对应范围的分区。
+
+* `dynamic_partition.prefix`
+
+ 动态创建的分区名前缀。
+
+* `dynamic_partition.buckets`
+
+ 动态创建的分区所对应的分桶数量。
-### 开启动态分区功能
-1. 首先需要在fe.conf中设置`dynamic_partition_enable=true`，可以在集群启动时通过修改配置文件指定，或者通过MySQL连接后使用命令行 `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`修改，也可以在运行时通过http接口动态修改,修改方法查看高级操作部分
+* `dynamic_partition.start_day_of_week`
-2. 如果需要对0.12版本之前的表添加动态分区属性，则需要通过以下命令修改表的属性
-```
-ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32");
-```
+ 当 `time_unit` 为 `WEEK` 时，该参数用于指定每周的起始点。取值为 1 到 7。其中 1 表示周一，7 表示周日。默认为 1，即表示每周以周一为起始点。
+
+* `dynamic_partition.start_day_of_month`
+
+ 当 `time_unit` 为 `MONTH` 时，该参数用于指定每月的起始日期。取值为 1 到 28。其中 1 表示每月1号，28 表示每月28号。默认为 1，即表示每月以1号位起始点。暂不支持以29、30、31号为起始日，以避免因闰年或闰月带来的歧义。
+
+## 示例
+
+1. 表 tbl1 分区列 k1 类型为 DATE，创建一个动态分区规则。按天分区，只保留最近7天的分区，并且预先创建未来3天的分区。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATE,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.start" = "-7",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "32"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29。则根据以上规则，tbl1 会产生以下分区：
+
+ ```
+ p20200529: ["2020-05-29", "2020-05-30")
+ p20200530: ["2020-05-30", "2020-05-31")
+ p20200531: ["2020-05-31", "2020-06-01")
+ p20200601: ["2020-06-01", "2020-06-02")
+ ```
+
+ 在第二天，即 2020-05-30，会创建新的分区 `p20200602: ["2020-06-02", "2020-06-03")`
+
+ 在 2020-06-06 时，因为 `dynamic_partition.start` 设置为 7，则将删除7天前的分区，即删除分区 `p20200529`。
+
+2. 表 tbl1 分区列 k1 类型为 DATETIME，创建一个动态分区规则。按星期分区，只保留最近2个星期的分区，并且预先创建未来2个星期的分区。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATETIME,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "WEEK",
+ "dynamic_partition.start" = "-2",
+ "dynamic_partition.end" = "2",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "8"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29，是 2020 年的第 22 周。默认每周起始为星期一。则根于以上规则，tbl1 会产生以下分区：
+
+ ```
+ p2020_22: ["2020-05-25 00:00:00", "2020-06-01 00:00:00")
+ p2020_23: ["2020-06-01 00:00:00", "2020-06-08 00:00:00")
+ p2020_24: ["2020-06-08 00:00:00", "2020-06-15 00:00:00")
+ ```
+
+ 其中每个分区的起始日期为当周的周一。同时，因为分区列 k1 的类型为 DATETIME，则分区值会补全时分秒部分，且皆为 0。
+
+ 在 2020-06-15，即第25周时，会删除2周前的分区，即删除 `p2020_22`。
+
+ 在上面的例子中，假设用户指定了周起始日为 `"dynamic_partition.start_day_of_week" = "3"`，即以每周三为起始日。则分区如下：
+
+ ```
+ p2020_22: ["2020-05-27 00:00:00", "2020-06-03 00:00:00")
+ p2020_23: ["2020-06-03 00:00:00", "2020-06-10 00:00:00")
+ p2020_24: ["2020-06-10 00:00:00", "2020-06-17 00:00:00")
+ ```
+
+ 即分区范围为当周的周三到下周的周二。
+
+ * 注：2019-12-31 和 2020-01-01 在同一周内，如果分区的起始日期为 2019-12-31，则分区名为 `p2019_53`，如果分区的起始日期为 2020-01-01，则分区名为 `p2020_01`。
+
+3. 表 tbl1 分区列 k1 类型为 DATE，创建一个动态分区规则。按月分区，不删除历史分区，并且预先创建未来2个月的分区。同时设定以每月3号为起始日。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATE,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "MONTH",
+ "dynamic_partition.end" = "2",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "8",
+ "dynamic_partition.start_day_of_month" = "3"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29。则根于以上规则，tbl1 会产生以下分区：
+
+ ```
+ p202005: ["2020-05-03", "2020-06-03")
+ p202006: ["2020-06-03", "2020-07-03")
+ p202007: ["2020-07-03", "2020-08-03")
+ ```
-### 停止动态分区功能
+ 因为没有设置 `dynamic_partition.start`，则不会删除历史分区。
+
+ 假设今天为 2020-05-20，并设置以每月28号为起始日，则分区范围为：
+
+ ```
+ p202004: ["2020-04-28", "2020-05-28")
+ p202005: ["2020-05-28", "2020-06-28")
+ p202006: ["2020-06-28", "2020-07-28")
+ ```
+
+## 修改动态分区属性
-如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=false`
+通过如下命令可以修改动态分区的属性：
-如果需要对指定表停止动态分区功能，则可以通过以下命令修改表的属性
```
-ALTER TABLE dynamic_partition SET ("dynamic_partition.enable" = "false")
+ALTER TABLE tbl1 SET
+(
+ "dynamic_partition.prop1" = "value1",
+ ...
+);
```
-### 修改动态分区属性
+某些属性的修改可能会可能会产生冲突。假设之前分区粒度为 DAY，并且已经创建了如下分区：
-通过如下命令可以修改动态分区的属性
```
-ALTER TABLE dynamic_partition SET ("key" = "value")
+p20200519: ["2020-05-19", "2020-05-20")
+p20200520: ["2020-05-20", "2020-05-21")
+p20200521: ["2020-05-21", "2020-05-22")
```
+如果此时将分区粒度改为 MONTH，则系统会尝试创建范围为 `["2020-05-01", "2020-06-01")` 的分区，而该分区的分区范围和已有分区冲突，所以无法创建。而范围为 `["2020-06-01", "2020-07-01")` 的分区可以正常创建。因此，2020-05-22 到 2020-05-30 时间段的分区，需要自行填补。
Review comment:
如果命令冲突时，该命令返回成功还是失败？
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429858538
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -41,117 +39,239 @@ under the License.
在某些使用场景下，用户会将表按照天进行分区划分，每天定时执行例行任务，这时需要使用方手动管理分区，否则可能由于使用方没有创建分区导致数据导入失败，这给使用方带来了额外的维护成本。
-在实现方式上, FE会启动一个后台线程，根据fe.conf中`dynamic_partition_enable` 及 `dynamic_partition_check_interval_seconds`参数决定该线程是否启动以及该线程的调度频率。每次调度时，会在注册表中读取动态分区表的属性，并根据动态分区属性动态添加及删除分区。
-
-建表时，在properties中指定dynamic_partition属性，FE首先对动态分区属性进行解析，校验输入参数的合法性，然后将对应的属性持久化到FE的元数据中，并将该表注册到动态分区列表中，后台线程会根据配置参数定期对动态分区列表进行扫描，读取表的动态分区属性，执行添加分区及删除分区的任务，每次的调度信息会保留在FE的内存中（重启后则丢失），可以通过`SHOW DYNAMIC PARTITION TABLES`查看调度任务是否成功，如果存在分区创建或删除失败，会将失败信息输出。
+通过动态分区功能，用户可以在建表时设定动态分区的规则。FE 会启动一个后台线程，根据用户指定的规则创建或删除分区。用户也可以在运行时对现有规则进行变更。
## 使用方式
-### 动态分区属性参数说明:
+动态分区的规则可以在建表时指定，或者在运行时进行修改。当前仅支持对单分区列的分区表设定动态分区规则。
+
+* 建表时指定：
+
+ ```
+ CREATE TABLE tbl1
+ (...)
+ PROPERTIES
+ (
+ "dynamic_partition.prop1" = "value1",
+ "dynamic_partition.prop2" = "value2",
+ ...
+ )
+ ```
-`dynamic_partition.enable`: 是否开启动态分区特性，可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。
+* 运行时修改
+ ```
+ ALTER TABLE tbl1 SET
+ (
+ "dynamic_partition.prop1" = "value1",
+ "dynamic_partition.prop2" = "value2",
+ ...
+ )
+ ```
-`dynamic_partition.time_unit`: 动态分区调度的单位，可指定为 `DAY` `WEEK` `MONTH`，当指定为 `DAY` 时，动态创建的分区名后缀格式为`yyyyMMdd`，例如`20200325`。当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
+### 动态分区规则参数
-`dynamic_partition.start`: 动态分区的开始时间, 以当天为基准，超过该时间范围的分区将会被删除。如果不填写，则默认为`Integer.MIN_VALUE` 即 `-2147483648`。
+动态分区的规则参数都以 `dynamic_partition.` 为前缀：
+* `dynamic_partition.enable`
-`dynamic_partition.end`: 动态分区的结束时间, 以当天为基准，会提前创建N个单位的分区范围。
+ 是否开启动态分区特性。可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。如果为 `FALSE`，则 Doris 会忽略该表的动态分区规则。
-`dynamic_partition.prefix`: 动态创建的分区名前缀。
+* `dynamic_partition.time_unit`
-`dynamic_partition.buckets`: 动态创建的分区所对应的分桶数量。
+ 动态分区调度的单位。可指定为 `DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。
-### 建表
+ 当指定为 `DAY` 时，动态创建的分区名后缀格式为 `yyyyMMdd`，例如`20200325`。
+
+ 当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`。即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。
+
+ 当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
-建表时，可以在 `PROPERTIES` 中指定以下`dynamic_partition`属性，表示这个表是一个动态分区表。
+* `dynamic_partition.start`
-示例：
+ 动态分区的起始偏移，为负数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，分区范围在此偏移之前的分区将会被删除。如果不填写，则默认为 `-2147483648`，即不删除历史分区。
+
+* `dynamic_partition.end`
-```
-CREATE TABLE example_db.dynamic_partition
-(
-k1 DATE,
-k2 INT,
-k3 SMALLINT,
-v1 VARCHAR(2048),
-v2 DATETIME DEFAULT "2014-02-04 15:36:00"
-)
-ENGINE=olap
-DUPLICATE KEY(k1, k2, k3)
-PARTITION BY RANGE (k1)
-(
-PARTITION p20200321 VALUES LESS THAN ("2020-03-22"),
-PARTITION p20200322 VALUES LESS THAN ("2020-03-23"),
-PARTITION p20200323 VALUES LESS THAN ("2020-03-24"),
-PARTITION p20200324 VALUES LESS THAN ("2020-03-25")
-)
-DISTRIBUTED BY HASH(k2) BUCKETS 32
-PROPERTIES(
-"storage_medium" = "SSD",
-"dynamic_partition.enable" = "true",
-"dynamic_partition.time_unit" = "DAY",
-"dynamic_partition.start" = "-3",
-"dynamic_partition.end" = "3",
-"dynamic_partition.prefix" = "p",
-"dynamic_partition.buckets" = "32"
- );
-```
-创建一张动态分区表，指定开启动态分区特性，以当天为2020-03-25为例，在每次调度时，会删除分区上界小于 `2020-03-22` 的分区，为了避免删除非动态创建的分区，动态删除分区只会删除分区名符合动态创建分区规则的分区，例如分区名为a1, 则即使分区范围在待删除的分区范围内，也不会被删除。同时在调度时会提前创建今天以及以后3天（总共4天）的分区(若分区已存在则会忽略)，分区名根据指定前缀分别为`p20200325` `p20200326` `p20200327` `p20200328`,每个分区的分桶数量为32。同时会删除 `p20200321` 的分区，最终的分区范围如下:
-```
-[types: [DATE]; keys: [2020-03-22]; ‥types: [DATE]; keys: [2020-03-23]; )
-[types: [DATE]; keys: [2020-03-23]; ‥types: [DATE]; keys: [2020-03-24]; )
-[types: [DATE]; keys: [2020-03-24]; ‥types: [DATE]; keys: [2020-03-25]; )
-[types: [DATE]; keys: [2020-03-25]; ‥types: [DATE]; keys: [2020-03-26]; )
-[types: [DATE]; keys: [2020-03-26]; ‥types: [DATE]; keys: [2020-03-27]; )
-[types: [DATE]; keys: [2020-03-27]; ‥types: [DATE]; keys: [2020-03-28]; )
-[types: [DATE]; keys: [2020-03-28]; ‥types: [DATE]; keys: [2020-03-29]; )
-```
+ 动态分区的结束偏移，为正数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，提前创建对应范围的分区。
+
+* `dynamic_partition.prefix`
+
+ 动态创建的分区名前缀。
+
+* `dynamic_partition.buckets`
+
+ 动态创建的分区所对应的分桶数量。
-### 开启动态分区功能
-1. 首先需要在fe.conf中设置`dynamic_partition_enable=true`，可以在集群启动时通过修改配置文件指定，或者通过MySQL连接后使用命令行 `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`修改，也可以在运行时通过http接口动态修改,修改方法查看高级操作部分
+* `dynamic_partition.start_day_of_week`
-2. 如果需要对0.12版本之前的表添加动态分区属性，则需要通过以下命令修改表的属性
-```
-ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32");
-```
+ 当 `time_unit` 为 `WEEK` 时，该参数用于指定每周的起始点。取值为 1 到 7。其中 1 表示周一，7 表示周日。默认为 1，即表示每周以周一为起始点。
+
+* `dynamic_partition.start_day_of_month`
+
+ 当 `time_unit` 为 `MONTH` 时，该参数用于指定每月的起始日期。取值为 1 到 28。其中 1 表示每月1号，28 表示每月28号。默认为 1，即表示每月以1号位起始点。暂不支持以29、30、31号为起始日，以避免因闰年或闰月带来的歧义。
+
+## 示例
+
+1. 表 tbl1 分区列 k1 类型为 DATE，创建一个动态分区规则。按天分区，只保留最近7天的分区，并且预先创建未来3天的分区。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATE,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.start" = "-7",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "32"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29。则根据以上规则，tbl1 会产生以下分区：
+
+ ```
+ p20200529: ["2020-05-29", "2020-05-30")
+ p20200530: ["2020-05-30", "2020-05-31")
+ p20200531: ["2020-05-31", "2020-06-01")
+ p20200601: ["2020-06-01", "2020-06-02")
+ ```
+
+ 在第二天，即 2020-05-30，会创建新的分区 `p20200602: ["2020-06-02", "2020-06-03")`
+
+ 在 2020-06-06 时，因为 `dynamic_partition.start` 设置为 7，则将删除7天前的分区，即删除分区 `p20200529`。
+
+2. 表 tbl1 分区列 k1 类型为 DATETIME，创建一个动态分区规则。按星期分区，只保留最近2个星期的分区，并且预先创建未来2个星期的分区。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATETIME,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "WEEK",
+ "dynamic_partition.start" = "-2",
+ "dynamic_partition.end" = "2",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "8"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29，是 2020 年的第 22 周。默认每周起始为星期一。则根于以上规则，tbl1 会产生以下分区：
+
+ ```
+ p2020_22: ["2020-05-25 00:00:00", "2020-06-01 00:00:00")
+ p2020_23: ["2020-06-01 00:00:00", "2020-06-08 00:00:00")
+ p2020_24: ["2020-06-08 00:00:00", "2020-06-15 00:00:00")
+ ```
+
+ 其中每个分区的起始日期为当周的周一。同时，因为分区列 k1 的类型为 DATETIME，则分区值会补全时分秒部分，且皆为 0。
+
+ 在 2020-06-15，即第25周时，会删除2周前的分区，即删除 `p2020_22`。
+
+ 在上面的例子中，假设用户指定了周起始日为 `"dynamic_partition.start_day_of_week" = "3"`，即以每周三为起始日。则分区如下：
+
+ ```
+ p2020_22: ["2020-05-27 00:00:00", "2020-06-03 00:00:00")
+ p2020_23: ["2020-06-03 00:00:00", "2020-06-10 00:00:00")
+ p2020_24: ["2020-06-10 00:00:00", "2020-06-17 00:00:00")
+ ```
+
+ 即分区范围为当周的周三到下周的周二。
+
+ * 注：2019-12-31 和 2020-01-01 在同一周内，如果分区的起始日期为 2019-12-31，则分区名为 `p2019_53`，如果分区的起始日期为 2020-01-01，则分区名为 `p2020_01`。
+
+3. 表 tbl1 分区列 k1 类型为 DATE，创建一个动态分区规则。按月分区，不删除历史分区，并且预先创建未来2个月的分区。同时设定以每月3号为起始日。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATE,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "MONTH",
+ "dynamic_partition.end" = "2",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "8",
+ "dynamic_partition.start_day_of_month" = "3"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29。则根于以上规则，tbl1 会产生以下分区：
+
+ ```
+ p202005: ["2020-05-03", "2020-06-03")
+ p202006: ["2020-06-03", "2020-07-03")
+ p202007: ["2020-07-03", "2020-08-03")
+ ```
-### 停止动态分区功能
+ 因为没有设置 `dynamic_partition.start`，则不会删除历史分区。
+
+ 假设今天为 2020-05-20，并设置以每月28号为起始日，则分区范围为：
+
+ ```
+ p202004: ["2020-04-28", "2020-05-28")
+ p202005: ["2020-05-28", "2020-06-28")
+ p202006: ["2020-06-28", "2020-07-28")
+ ```
+
+## 修改动态分区属性
-如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=false`
+通过如下命令可以修改动态分区的属性：
-如果需要对指定表停止动态分区功能，则可以通过以下命令修改表的属性
```
-ALTER TABLE dynamic_partition SET ("dynamic_partition.enable" = "false")
+ALTER TABLE tbl1 SET
+(
+ "dynamic_partition.prop1" = "value1",
+ ...
+);
```
-### 修改动态分区属性
+某些属性的修改可能会可能会产生冲突。假设之前分区粒度为 DAY，并且已经创建了如下分区：
-通过如下命令可以修改动态分区的属性
```
-ALTER TABLE dynamic_partition SET ("key" = "value")
+p20200519: ["2020-05-19", "2020-05-20")
+p20200520: ["2020-05-20", "2020-05-21")
+p20200521: ["2020-05-21", "2020-05-22")
```
+如果此时将分区粒度改为 MONTH，则系统会尝试创建范围为 `["2020-05-01", "2020-06-01")` 的分区，而该分区的分区范围和已有分区冲突，所以无法创建。而范围为 `["2020-06-01", "2020-07-01")` 的分区可以正常创建。因此，2020-05-22 到 2020-05-30 时间段的分区，需要自行填补。
Review comment:
目前是直接成功，不会报错，show dynamic partition 可以看到错误信息。
这个我建议由用户自行处理。系统来处理这个问题，太复杂了，容易出错，并且隐含的操作太多。
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#issuecomment-633972342
LGTM
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429850385
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
+ current.add(Calendar.DATE, (startOf.dayOfWeek - day));
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in MONTH granularity.
+ * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month.
+ * startOf: Define the start date of each month. 1 means start on the 1st of every month.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = 1, startOf.month = 3
+ * It will return 2020-06-03
+ */
+ private static String getPartitionRangeOfMonth(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
Review comment:
What about use `LocalDateTime/ZonedDateTime` to replace all `Calendar`? They are thread-safe and support timezone
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429858691
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -161,48 +281,26 @@ SHOW DYNAMIC PARTITION TABLES;
* dynamic\_partition\_enable
- 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。
+ 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。可以通过修改 fe.conf 中的参数并重启 FE 生效。也可以在运行时执行以下命令生效：
-* dynamic\_partition\_check\_interval\_seconds
-
- 动态分区线程的执行频率，默认为3600(1个小时)，即每1个小时进行一次调度
+ MySQL 协议：
-### HTTP Restful API
-
-Doris 提供了修改动态分区配置参数的 HTTP Restful API，用于运行时修改动态分区配置参数。
-
-该 API 实现在 FE 端，使用 `fe_host:fe_http_port` 进行访问。需要 ADMIN 权限。
-
-1. 将 dynamic_partition_enable 设置为 true 或 false
+ `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`
+
+ HTTP 协议：
+
+ `curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true`
- * 标记为 true
+ 若要全局关闭动态分区，则设置此参数为 false 即可。
- ```
- GET /api/_set_config?dynamic_partition_enable=true
-
- 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true
-
- 返回：200
- ```
-
- * 标记为 false
+* dynamic\_partition\_check\_interval\_seconds
+
+ 动态分区线程的执行频率，默认为3600(1个小时)，即每1个小时进行一次调度。可以通过修改 fe.conf 中的参数并重启 FE 生效。也可以在运行时执行以下命令修改：
- ```
- GET /api/_set_config?dynamic_partition_enable=false
-
- 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false
-
- 返回：200
- ```
+ MySQL 协议：
+
+ `ADMIN SET FRONTEND CONFIG ("dynamic_partition_check_interval_seconds" = "7200")`
-2. 设置 dynamic partition 的调度频率
+ HTTP 协议：
- * 设置调度时间为12小时调度一次
-
- ```
- GET /api/_set_config?dynamic_partition_check_interval_seconds=432000
-
- 例如: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000
-
- 返回：200
- ```
+ `curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000`
Review comment:
这个就是沿用之前的框架，没有做更多的修改
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429852489
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
Review comment:
Use LocalDateTime can avoid this trick
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429937221
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -210,7 +215,8 @@ public void createOrUpdateRuntimeInfo(String tableName, String key, String value
reservePartitionKeyRange = Range.closedOpen(lowerBound, upperBound);
} catch (AnalysisException e) {
// keys.size is always equal to column.size, cannot reach this exception
- LOG.warn("Keys size is not equal to column size. Error={}", e.getMessage());
+ LOG.warn("Keys size is not equal to column size. Error={}, db: {}, table: {}", e.getMessage(),
+ db.getFullName(), olapTable.getName());
Review comment:
OK, I will check this offset to make it reasonable.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429857971
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -161,48 +281,26 @@ SHOW DYNAMIC PARTITION TABLES;
* dynamic\_partition\_enable
- 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。
+ 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。可以通过修改 fe.conf 中的参数并重启 FE 生效。也可以在运行时执行以下命令生效：
Review comment:
我打算等0.12版本稳定了再打开这个限制，目前来看，还是有各种各样的小问题。。
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wutiangan commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
wutiangan commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429810367
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
+ current.add(Calendar.DATE, (startOf.dayOfWeek - day));
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in MONTH granularity.
+ * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month.
+ * startOf: Define the start date of each month. 1 means start on the 1st of every month.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = 1, startOf.month = 3
+ * It will return 2020-06-03
+ */
+ private static String getPartitionRangeOfMonth(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
Review comment:
why tz is no use?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429950455
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -210,7 +215,8 @@ public void createOrUpdateRuntimeInfo(String tableName, String key, String value
reservePartitionKeyRange = Range.closedOpen(lowerBound, upperBound);
} catch (AnalysisException e) {
// keys.size is always equal to column.size, cannot reach this exception
- LOG.warn("Keys size is not equal to column size. Error={}", e.getMessage());
+ LOG.warn("Keys size is not equal to column size. Error={}, db: {}, table: {}", e.getMessage(),
+ db.getFullName(), olapTable.getName());
Review comment:
I just make a check that if `start offset == Integer.MIN_VALUE`, it will not drop any partition.
And for adding partition, I just leave it to user to make sure the value is reasonable.
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -114,6 +118,34 @@ private static void checkEnable(String enable) throws DdlException {
}
}
+ private static void checkStartDayOfMonth(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
+ }
+ try {
+ int dayOfMonth = Integer.parseInt(val);
+ if (dayOfMonth < 1 || dayOfMonth > 28) {
Review comment:
I have already stated the reason in the document, I will add some comments here.
##########
File path: fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
##########
@@ -268,4 +269,11 @@ public static String checkTimeZoneValidAndStandardize(String value) throws DdlEx
}
throw new DdlException("Parse time zone " + value + " error");
}
+
+ // return the number of days of give year.month
+ public static int getNumberDaysOfMonth(int year, int month) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.set(year, month + 1, 0);
Review comment:
This function is useless, i will remove it
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429936606
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -275,19 +280,25 @@ private void executeDynamicPartition() {
// scheduler time should be record even no partition added
createOrUpdateRuntimeInfo(olapTable.getName(), LAST_SCHEDULER_TIME, TimeUtils.getCurrentFormatTime());
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo();
+ if (rangePartitionInfo.getPartitionColumns().size() != 1) {
+ // currently only support partition with single column.
+ iterator.remove();
+ continue;
+ }
+
Review comment:
It is just a self-defence~
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman merged pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429885387
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
+ current.add(Calendar.DATE, (startOf.dayOfWeek - day));
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in MONTH granularity.
+ * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month.
+ * startOf: Define the start date of each month. 1 means start on the 1st of every month.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = 1, startOf.month = 3
+ * It will return 2020-06-03
+ */
+ private static String getPartitionRangeOfMonth(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
Review comment:
OK, we can change it in next PR~
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429897947
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -114,6 +118,34 @@ private static void checkEnable(String enable) throws DdlException {
}
}
+ private static void checkStartDayOfMonth(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
+ }
+ try {
+ int dayOfMonth = Integer.parseInt(val);
+ if (dayOfMonth < 1 || dayOfMonth > 28) {
Review comment:
add a comment why less than 28, the reason in doc is for users?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wutiangan commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
wutiangan commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429803209
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -41,117 +39,239 @@ under the License.
在某些使用场景下，用户会将表按照天进行分区划分，每天定时执行例行任务，这时需要使用方手动管理分区，否则可能由于使用方没有创建分区导致数据导入失败，这给使用方带来了额外的维护成本。
-在实现方式上, FE会启动一个后台线程，根据fe.conf中`dynamic_partition_enable` 及 `dynamic_partition_check_interval_seconds`参数决定该线程是否启动以及该线程的调度频率。每次调度时，会在注册表中读取动态分区表的属性，并根据动态分区属性动态添加及删除分区。
-
-建表时，在properties中指定dynamic_partition属性，FE首先对动态分区属性进行解析，校验输入参数的合法性，然后将对应的属性持久化到FE的元数据中，并将该表注册到动态分区列表中，后台线程会根据配置参数定期对动态分区列表进行扫描，读取表的动态分区属性，执行添加分区及删除分区的任务，每次的调度信息会保留在FE的内存中（重启后则丢失），可以通过`SHOW DYNAMIC PARTITION TABLES`查看调度任务是否成功，如果存在分区创建或删除失败，会将失败信息输出。
+通过动态分区功能，用户可以在建表时设定动态分区的规则。FE 会启动一个后台线程，根据用户指定的规则创建或删除分区。用户也可以在运行时对现有规则进行变更。
## 使用方式
-### 动态分区属性参数说明:
+动态分区的规则可以在建表时指定，或者在运行时进行修改。当前仅支持对单分区列的分区表设定动态分区规则。
+
+* 建表时指定：
+
+ ```
+ CREATE TABLE tbl1
+ (...)
+ PROPERTIES
+ (
+ "dynamic_partition.prop1" = "value1",
+ "dynamic_partition.prop2" = "value2",
+ ...
+ )
+ ```
-`dynamic_partition.enable`: 是否开启动态分区特性，可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。
+* 运行时修改
+ ```
+ ALTER TABLE tbl1 SET
+ (
+ "dynamic_partition.prop1" = "value1",
+ "dynamic_partition.prop2" = "value2",
+ ...
+ )
+ ```
-`dynamic_partition.time_unit`: 动态分区调度的单位，可指定为 `DAY` `WEEK` `MONTH`，当指定为 `DAY` 时，动态创建的分区名后缀格式为`yyyyMMdd`，例如`20200325`。当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
+### 动态分区规则参数
-`dynamic_partition.start`: 动态分区的开始时间, 以当天为基准，超过该时间范围的分区将会被删除。如果不填写，则默认为`Integer.MIN_VALUE` 即 `-2147483648`。
+动态分区的规则参数都以 `dynamic_partition.` 为前缀：
+* `dynamic_partition.enable`
-`dynamic_partition.end`: 动态分区的结束时间, 以当天为基准，会提前创建N个单位的分区范围。
+ 是否开启动态分区特性。可指定为 `TRUE` 或 `FALSE`。如果不填写，默认为 `TRUE`。如果为 `FALSE`，则 Doris 会忽略该表的动态分区规则。
-`dynamic_partition.prefix`: 动态创建的分区名前缀。
+* `dynamic_partition.time_unit`
-`dynamic_partition.buckets`: 动态创建的分区所对应的分桶数量。
+ 动态分区调度的单位。可指定为 `DAY`、`WEEK`、`MONTH`。分别表示按天、按星期、按月进行分区创建或删除。
-### 建表
+ 当指定为 `DAY` 时，动态创建的分区名后缀格式为 `yyyyMMdd`，例如`20200325`。
+
+ 当指定为 `WEEK` 时，动态创建的分区名后缀格式为`yyyy_ww`。即当前日期属于这一年的第几周，例如 `2020-03-25` 创建的分区名后缀为 `2020_13`, 表明目前为2020年第13周。
+
+ 当指定为 `MONTH` 时，动态创建的分区名后缀格式为 `yyyyMM`，例如 `202003`。
-建表时，可以在 `PROPERTIES` 中指定以下`dynamic_partition`属性，表示这个表是一个动态分区表。
+* `dynamic_partition.start`
-示例：
+ 动态分区的起始偏移，为负数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，分区范围在此偏移之前的分区将会被删除。如果不填写，则默认为 `-2147483648`，即不删除历史分区。
+
+* `dynamic_partition.end`
-```
-CREATE TABLE example_db.dynamic_partition
-(
-k1 DATE,
-k2 INT,
-k3 SMALLINT,
-v1 VARCHAR(2048),
-v2 DATETIME DEFAULT "2014-02-04 15:36:00"
-)
-ENGINE=olap
-DUPLICATE KEY(k1, k2, k3)
-PARTITION BY RANGE (k1)
-(
-PARTITION p20200321 VALUES LESS THAN ("2020-03-22"),
-PARTITION p20200322 VALUES LESS THAN ("2020-03-23"),
-PARTITION p20200323 VALUES LESS THAN ("2020-03-24"),
-PARTITION p20200324 VALUES LESS THAN ("2020-03-25")
-)
-DISTRIBUTED BY HASH(k2) BUCKETS 32
-PROPERTIES(
-"storage_medium" = "SSD",
-"dynamic_partition.enable" = "true",
-"dynamic_partition.time_unit" = "DAY",
-"dynamic_partition.start" = "-3",
-"dynamic_partition.end" = "3",
-"dynamic_partition.prefix" = "p",
-"dynamic_partition.buckets" = "32"
- );
-```
-创建一张动态分区表，指定开启动态分区特性，以当天为2020-03-25为例，在每次调度时，会删除分区上界小于 `2020-03-22` 的分区，为了避免删除非动态创建的分区，动态删除分区只会删除分区名符合动态创建分区规则的分区，例如分区名为a1, 则即使分区范围在待删除的分区范围内，也不会被删除。同时在调度时会提前创建今天以及以后3天（总共4天）的分区(若分区已存在则会忽略)，分区名根据指定前缀分别为`p20200325` `p20200326` `p20200327` `p20200328`,每个分区的分桶数量为32。同时会删除 `p20200321` 的分区，最终的分区范围如下:
-```
-[types: [DATE]; keys: [2020-03-22]; ‥types: [DATE]; keys: [2020-03-23]; )
-[types: [DATE]; keys: [2020-03-23]; ‥types: [DATE]; keys: [2020-03-24]; )
-[types: [DATE]; keys: [2020-03-24]; ‥types: [DATE]; keys: [2020-03-25]; )
-[types: [DATE]; keys: [2020-03-25]; ‥types: [DATE]; keys: [2020-03-26]; )
-[types: [DATE]; keys: [2020-03-26]; ‥types: [DATE]; keys: [2020-03-27]; )
-[types: [DATE]; keys: [2020-03-27]; ‥types: [DATE]; keys: [2020-03-28]; )
-[types: [DATE]; keys: [2020-03-28]; ‥types: [DATE]; keys: [2020-03-29]; )
-```
+ 动态分区的结束偏移，为正数。根据 `time_unit` 属性的不同，以当天（星期/月）为基准，提前创建对应范围的分区。
+
+* `dynamic_partition.prefix`
+
+ 动态创建的分区名前缀。
+
+* `dynamic_partition.buckets`
+
+ 动态创建的分区所对应的分桶数量。
-### 开启动态分区功能
-1. 首先需要在fe.conf中设置`dynamic_partition_enable=true`，可以在集群启动时通过修改配置文件指定，或者通过MySQL连接后使用命令行 `ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true")`修改，也可以在运行时通过http接口动态修改,修改方法查看高级操作部分
+* `dynamic_partition.start_day_of_week`
-2. 如果需要对0.12版本之前的表添加动态分区属性，则需要通过以下命令修改表的属性
-```
-ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32");
-```
+ 当 `time_unit` 为 `WEEK` 时，该参数用于指定每周的起始点。取值为 1 到 7。其中 1 表示周一，7 表示周日。默认为 1，即表示每周以周一为起始点。
+
+* `dynamic_partition.start_day_of_month`
+
+ 当 `time_unit` 为 `MONTH` 时，该参数用于指定每月的起始日期。取值为 1 到 28。其中 1 表示每月1号，28 表示每月28号。默认为 1，即表示每月以1号位起始点。暂不支持以29、30、31号为起始日，以避免因闰年或闰月带来的歧义。
+
+## 示例
+
+1. 表 tbl1 分区列 k1 类型为 DATE，创建一个动态分区规则。按天分区，只保留最近7天的分区，并且预先创建未来3天的分区。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATE,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.start" = "-7",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "32"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29。则根据以上规则，tbl1 会产生以下分区：
+
+ ```
+ p20200529: ["2020-05-29", "2020-05-30")
+ p20200530: ["2020-05-30", "2020-05-31")
+ p20200531: ["2020-05-31", "2020-06-01")
+ p20200601: ["2020-06-01", "2020-06-02")
+ ```
+
+ 在第二天，即 2020-05-30，会创建新的分区 `p20200602: ["2020-06-02", "2020-06-03")`
+
+ 在 2020-06-06 时，因为 `dynamic_partition.start` 设置为 7，则将删除7天前的分区，即删除分区 `p20200529`。
+
+2. 表 tbl1 分区列 k1 类型为 DATETIME，创建一个动态分区规则。按星期分区，只保留最近2个星期的分区，并且预先创建未来2个星期的分区。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATETIME,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "WEEK",
+ "dynamic_partition.start" = "-2",
+ "dynamic_partition.end" = "2",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "8"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29，是 2020 年的第 22 周。默认每周起始为星期一。则根于以上规则，tbl1 会产生以下分区：
+
+ ```
+ p2020_22: ["2020-05-25 00:00:00", "2020-06-01 00:00:00")
+ p2020_23: ["2020-06-01 00:00:00", "2020-06-08 00:00:00")
+ p2020_24: ["2020-06-08 00:00:00", "2020-06-15 00:00:00")
+ ```
+
+ 其中每个分区的起始日期为当周的周一。同时，因为分区列 k1 的类型为 DATETIME，则分区值会补全时分秒部分，且皆为 0。
+
+ 在 2020-06-15，即第25周时，会删除2周前的分区，即删除 `p2020_22`。
+
+ 在上面的例子中，假设用户指定了周起始日为 `"dynamic_partition.start_day_of_week" = "3"`，即以每周三为起始日。则分区如下：
+
+ ```
+ p2020_22: ["2020-05-27 00:00:00", "2020-06-03 00:00:00")
+ p2020_23: ["2020-06-03 00:00:00", "2020-06-10 00:00:00")
+ p2020_24: ["2020-06-10 00:00:00", "2020-06-17 00:00:00")
+ ```
+
+ 即分区范围为当周的周三到下周的周二。
+
+ * 注：2019-12-31 和 2020-01-01 在同一周内，如果分区的起始日期为 2019-12-31，则分区名为 `p2019_53`，如果分区的起始日期为 2020-01-01，则分区名为 `p2020_01`。
+
+3. 表 tbl1 分区列 k1 类型为 DATE，创建一个动态分区规则。按月分区，不删除历史分区，并且预先创建未来2个月的分区。同时设定以每月3号为起始日。
+
+ ```
+ CREATA TABLE tbl1
+ (
+ k1 DATE,
+ ...
+ )
+ PARTITION BY RANGE(K1) ()
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "MONTH",
+ "dynamic_partition.end" = "2",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "8",
+ "dynamic_partition.start_day_of_month" = "3"
+ );
+ ```
+
+ 假设当前日期为 2020-05-29。则根于以上规则，tbl1 会产生以下分区：
+
+ ```
+ p202005: ["2020-05-03", "2020-06-03")
+ p202006: ["2020-06-03", "2020-07-03")
+ p202007: ["2020-07-03", "2020-08-03")
+ ```
-### 停止动态分区功能
+ 因为没有设置 `dynamic_partition.start`，则不会删除历史分区。
+
+ 假设今天为 2020-05-20，并设置以每月28号为起始日，则分区范围为：
+
+ ```
+ p202004: ["2020-04-28", "2020-05-28")
+ p202005: ["2020-05-28", "2020-06-28")
+ p202006: ["2020-06-28", "2020-07-28")
+ ```
+
+## 修改动态分区属性
-如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=false`
+通过如下命令可以修改动态分区的属性：
-如果需要对指定表停止动态分区功能，则可以通过以下命令修改表的属性
```
-ALTER TABLE dynamic_partition SET ("dynamic_partition.enable" = "false")
+ALTER TABLE tbl1 SET
+(
+ "dynamic_partition.prop1" = "value1",
+ ...
+);
```
-### 修改动态分区属性
+某些属性的修改可能会可能会产生冲突。假设之前分区粒度为 DAY，并且已经创建了如下分区：
-通过如下命令可以修改动态分区的属性
```
-ALTER TABLE dynamic_partition SET ("key" = "value")
+p20200519: ["2020-05-19", "2020-05-20")
+p20200520: ["2020-05-20", "2020-05-21")
+p20200521: ["2020-05-21", "2020-05-22")
```
+如果此时将分区粒度改为 MONTH，则系统会尝试创建范围为 `["2020-05-01", "2020-06-01")` 的分区，而该分区的分区范围和已有分区冲突，所以无法创建。而范围为 `["2020-06-01", "2020-07-01")` 的分区可以正常创建。因此，2020-05-22 到 2020-05-30 时间段的分区，需要自行填补。
Review comment:
如果命令冲突时，该命令返回成功还是失败？如果返回成功，那些需要补齐的分区能否自动帮助用户补齐？
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429897271
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -275,19 +280,25 @@ private void executeDynamicPartition() {
// scheduler time should be record even no partition added
createOrUpdateRuntimeInfo(olapTable.getName(), LAST_SCHEDULER_TIME, TimeUtils.getCurrentFormatTime());
RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo();
+ if (rangePartitionInfo.getPartitionColumns().size() != 1) {
+ // currently only support partition with single column.
+ iterator.remove();
+ continue;
+ }
+
Review comment:
The single column check work is done when create table, no need to check here
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429896775
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -149,10 +148,11 @@ public void createOrUpdateRuntimeInfo(String tableName, String key, String value
addPartitionKeyRange = Range.closedOpen(lowerBound, upperBound);
} catch (AnalysisException e) {
// keys.size is always equal to column.size, cannot reach this exception
- LOG.warn("Keys size is not equal to column size. Error={}", e.getMessage());
+ LOG.warn("Keys size is not equal to column size. Error={}, db: {}, table: {}", e.getMessage(),
+ db.getFullName(), olapTable.getName());
Review comment:
same as follower comment
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429805830
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -114,6 +118,34 @@ private static void checkEnable(String enable) throws DdlException {
}
}
+ private static void checkStartDayOfMonth(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
+ }
+ try {
+ int dayOfMonth = Integer.parseInt(val);
+ if (dayOfMonth < 1 || dayOfMonth > 28) {
+ throw new DdlException(DynamicPartitionProperty.START_DAY_OF_MONTH + " should between 1 and 28");
+ }
+ } catch (NumberFormatException e) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_MONTH);
+ }
+ }
+
+ private static void checkStartDayOfWeek(String val) throws DdlException {
+ if (Strings.isNullOrEmpty(val)) {
+ throw new DdlException("Invalid properties: " + DynamicPartitionProperty.START_DAY_OF_WEEK);
+ }
+ try {
+ int dayOfMonth = Integer.parseInt(val);
Review comment:
dayOfWeek
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r430122762
##########
File path: fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
##########
@@ -268,4 +269,11 @@ public static String checkTimeZoneValidAndStandardize(String value) throws DdlEx
}
throw new DdlException("Parse time zone " + value + " error");
}
+
+ // return the number of days of give year.month
+ public static int getNumberDaysOfMonth(int year, int month) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.set(year, month + 1, 0);
Review comment:
why plus one? `getNumberDaysOfMonth(2020, 1)` will return 29, and the function name is confused
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -149,10 +148,11 @@ public void createOrUpdateRuntimeInfo(String tableName, String key, String value
addPartitionKeyRange = Range.closedOpen(lowerBound, upperBound);
} catch (AnalysisException e) {
// keys.size is always equal to column.size, cannot reach this exception
- LOG.warn("Keys size is not equal to column size. Error={}", e.getMessage());
+ LOG.warn("Keys size is not equal to column size. Error={}, db: {}, table: {}", e.getMessage(),
+ db.getFullName(), olapTable.getName());
Review comment:
same as following comment
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429850385
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
+ * return formatted string of partition range in DAY granularity.
+ * offset: The offset from the current day. 0 means current day, 1 means tomorrow, -1 means yesterday.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1
+ * It will return 2020-05-23
+ */
+ private static String getPartitionRangeOfDay(Calendar current, int offset, TimeZone tz, String format) {
+ current.add(Calendar.DATE, offset);
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in WEEK granularity.
+ * offset: The offset from the current week. 0 means current week, 1 means next week, -1 means last week.
+ * startOf: Define the start day of each week. 1 means MONDAY, 7 means SUNDAY.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = -1, startOf.dayOfWeek = 3
+ * It will return 2020-05-20 (Wednesday of last week)
+ */
+ private static String getPartitionRangeOfWeek(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
+ String format) {
+ Preconditions.checkArgument(startOf.isStartOfWeek());
+ // 1. get the offset week
+ current.add(Calendar.WEEK_OF_YEAR, offset);
+ // 2. get the date of `startOf` week
+ int day = current.get(Calendar.DAY_OF_WEEK);
+ // SUNDAY will return 1, we will set it to 7, and make MONDAY to 1, and so on
+ day = day == 1 ? 7 : day - 1;
+ current.add(Calendar.DATE, (startOf.dayOfWeek - day));
+ return getFormattedTimeWithoutHourMinuteSecond(current, format);
+ }
+
+ /*
+ * return formatted string of partition range in MONTH granularity.
+ * offset: The offset from the current month. 0 means current month, 1 means next month, -1 means last month.
+ * startOf: Define the start date of each month. 1 means start on the 1st of every month.
+ * format: the format of the return date string.
+ *
+ * Eg:
+ * Today is 2020-05-24, offset = 1, startOf.month = 3
+ * It will return 2020-06-03
+ */
+ private static String getPartitionRangeOfMonth(Calendar current, int offset, StartOfDate startOf, TimeZone tz,
Review comment:
What about use `LocalDateTime/ZonedDateTime` to replace all `Calendar`? They are thread-safe, immutable and support timezone
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429862055
##########
File path: fe/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
##########
@@ -210,7 +215,8 @@ public void createOrUpdateRuntimeInfo(String tableName, String key, String value
reservePartitionKeyRange = Range.closedOpen(lowerBound, upperBound);
} catch (AnalysisException e) {
// keys.size is always equal to column.size, cannot reach this exception
- LOG.warn("Keys size is not equal to column size. Error={}", e.getMessage());
+ LOG.warn("Keys size is not equal to column size. Error={}, db: {}, table: {}", e.getMessage(),
+ db.getFullName(), olapTable.getName());
Review comment:
maybe we should remove the warning, because dynamic_partition.start is default to Integer.MIN_VALUE, so we may got a AnalysisException like this `Error=errCode = 2, detailMessage = date literal [5877471-07-26 00:00:00] is invalid`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429878339
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
Review comment:
It looks good.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429878339
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
Review comment:
It look good.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] WingsGo commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
WingsGo commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429800488
##########
File path: docs/zh-CN/administrator-guide/dynamic-partition.md
##########
@@ -161,48 +281,26 @@ SHOW DYNAMIC PARTITION TABLES;
* dynamic\_partition\_enable
- 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。
+ 是否开启 Doris 的动态分区功能。默认为 false，即关闭。该参数只影响动态分区表的分区操作，不影响普通表。可以通过修改 fe.conf 中的参数并重启 FE 生效。也可以在运行时执行以下命令生效：
Review comment:
这里我们是否可以放开默认为true?目前已经存在了比较长的时间了，有些用户可能不仔细看文档，创建了分区表但是没有修改配置，造成使用上的一些额外成本？
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman commented on a change in pull request #3679: [DynamicPartition] Optimize the rule of creating dynamic partition
Posted by GitBox <gi...@apache.org>.
morningman commented on a change in pull request #3679:
URL: https://github.com/apache/incubator-doris/pull/3679#discussion_r429860375
##########
File path: fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
##########
@@ -273,34 +321,110 @@ public static String getPartitionFormat(Column column) throws DdlException {
}
}
- public static String getFormattedPartitionName(String name, String timeUnit) {
- name = name.replace("-", "").replace(":", "").replace(" ", "");
+ public static String getFormattedPartitionName(TimeZone tz, String formattedDateStr, String timeUnit) {
+ formattedDateStr = formattedDateStr.replace("-", "").replace(":", "").replace(" ", "");
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- return name.substring(0, 8);
+ return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
- return name.substring(0, 6);
+ return formattedDateStr.substring(0, 6);
} else {
- name = name.substring(0, 8);
- Calendar calendar = Calendar.getInstance();
+ formattedDateStr = formattedDateStr.substring(0, 8);
+ Calendar calendar = Calendar.getInstance(tz);
try {
- calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(name));
+ calendar.setTime(new SimpleDateFormat("yyyyMMdd").parse(formattedDateStr));
} catch (ParseException e) {
LOG.warn("Format dynamic partition name error. Error={}", e.getMessage());
- return name;
+ return formattedDateStr;
+ }
+ int weekOfYear = calendar.get(Calendar.WEEK_OF_YEAR);
+ if (weekOfYear <= 1 && calendar.get(Calendar.MONTH) >= 11) {
+ // eg: JDK think 2019-12-30 as the first week of year 2020, we need to handle this.
+ // to make it as the 53rd week of year 2019.
+ weekOfYear += 52;
}
- return String.format("%s_%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.WEEK_OF_YEAR));
+ return String.format("%s_%02d", calendar.get(Calendar.YEAR), weekOfYear);
}
}
- public static String getPartitionRange(String timeUnit, int offset, Calendar calendar, String format) {
+ // return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
+ // TODO: support HOUR and YEAR
+ public static String getPartitionRangeString(DynamicPartitionProperty property, Calendar current,
+ int offset, String format) {
+ String timeUnit = property.getTimeUnit();
+ TimeZone tz = property.getTimeZone();
if (timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())) {
- calendar.add(Calendar.DAY_OF_MONTH, offset);
+ return getPartitionRangeOfDay(current, offset, tz, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
- calendar.add(Calendar.WEEK_OF_MONTH, offset);
- } else {
- calendar.add(Calendar.MONTH, offset);
+ return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), tz, format);
+ } else { // MONTH
+ return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), tz, format);
+ }
+ }
+
+ /*
Review comment:
Ok, I will change my IDE's setting. But there are lots of comment write like this. I can just make sure that newly added code can obey to rule.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org