You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by GitBox <gi...@apache.org> on 2018/11/27 01:40:15 UTC
[GitHub] Wayne1c closed pull request #356: KYLIN-3665 Partition time column
may never be used
Wayne1c closed pull request #356: KYLIN-3665 Partition time column may never be used
URL: https://github.com/apache/kylin/pull/356
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index dcb37ecdad..47802d9831 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -95,6 +95,14 @@ public boolean partitionColumnIsTimeMillis() {
return type.isBigInt() && !DateFormat.isDatePattern(partitionDateFormat);
}
+ public boolean partitionTimeColumnIsInt() {
+ if (partitionTimeColumnRef == null)
+ return false;
+
+ DataType type = partitionTimeColumnRef.getType();
+ return (type.isInt() || type.isBigInt());
+ }
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -194,103 +202,99 @@ public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, Segm
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
+ if (endExclusive <= startInclusive) {
+ return "1=0";
+ }
+
TblColRef partitionDateColumn = partDesc.getPartitionDateColumnRef();
TblColRef partitionTimeColumn = partDesc.getPartitionTimeColumnRef();
StringBuilder builder = new StringBuilder();
- if (partDesc.partitionColumnIsYmdInt()) {
- buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive,
- partDesc.getPartitionDateFormat());
- } else if (partDesc.partitionColumnIsTimeMillis()) {
- buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumn, startInclusive, endExclusive);
- } else if (partitionDateColumn != null && partitionTimeColumn == null) {
- buildSingleColumnRangeCondition(builder, partitionDateColumn, startInclusive, endExclusive,
- partDesc.getPartitionDateFormat());
- } else if (partitionDateColumn == null && partitionTimeColumn != null) {
- buildSingleColumnRangeCondition(builder, partitionTimeColumn, startInclusive, endExclusive,
- partDesc.getPartitionTimeFormat());
- } else if (partitionDateColumn != null && partitionTimeColumn != null) {
- buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive,
- endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat());
+ if (partitionDateColumn != null && partitionTimeColumn != null) {
+ buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc);
+ } else if (partitionDateColumn != null) {
+ buildSingleColumnRangeCondition(builder, partitionDateColumn, startInclusive, endExclusive, partDesc, true);
+ } else if (partitionTimeColumn != null) {
+ buildSingleColumnRangeCondition(builder, partitionTimeColumn, startInclusive, endExclusive, partDesc, false);
}
return builder.toString();
}
- private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getIdentity();
- builder.append(partitionColumnName + " >= " + startInclusive);
- builder.append(" AND ");
- builder.append(partitionColumnName + " < " + endExclusive);
- }
-
- private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive, String partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getIdentity();
- builder.append(partitionColumnName + " >= "
- + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat));
- builder.append(" AND ");
- builder.append(
- partitionColumnName + " < " + DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat));
- }
-
private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive, String partitionColumnDateFormat) {
+ long startInclusive, long endExclusive, PartitionDesc partitionDesc, boolean isPartitionDateColumn) {
String partitionColumnName = partitionColumn.getIdentity();
- if (endExclusive <= startInclusive) {
- builder.append("1=0");
- return;
- }
-
String startInc = null;
String endInc = null;
- if (StringUtils.isBlank(partitionColumnDateFormat)) {
- startInc = String.valueOf(startInclusive);
- endInc = String.valueOf(endExclusive);
+
+ if (isPartitionDateColumn) {
+ startInc = convertDateConditionValue(startInclusive, partitionDesc);
+ endInc = convertDateConditionValue(endExclusive, partitionDesc);
} else {
- startInc = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
- endInc = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
+ startInc = convertTimeConditionValue(startInclusive, partitionDesc);
+ endInc = convertTimeConditionValue(endExclusive, partitionDesc);
}
- builder.append(partitionColumnName + " >= '" + startInc + "'");
+ builder.append(partitionColumnName + " >= " + startInc);
builder.append(" AND ");
- builder.append(partitionColumnName + " < '" + endInc + "'");
+ builder.append(partitionColumnName + " < " + endInc);
+ }
+
+ private static String convertDateConditionValue(long date, PartitionDesc partitionDesc) {
+ if(partitionDesc.partitionColumnIsYmdInt()) {
+ return DateFormat.formatToDateStr(date, partitionDesc.getPartitionDateFormat());
+ } else if (partitionDesc.partitionColumnIsTimeMillis()) {
+ return String.valueOf(date);
+ } else {
+ return "'" + DateFormat.formatToDateStr(date, partitionDesc.getPartitionDateFormat()) + "'";
+ }
+ }
+
+ private static String convertTimeConditionValue(long time, PartitionDesc partitionDesc) {
+ //currently supported time format: HH:mm:ss、HH:mm、HH(String/int)
+ //TODO: HHmmss、HHmm(String/int)
+
+ if (partitionDesc.partitionTimeColumnIsInt()) {
+ return DateFormat.formatToDateStr(time, partitionDesc.getPartitionTimeFormat());
+ } else {
+ return "'" + DateFormat.formatToDateStr(time, partitionDesc.getPartitionTimeFormat()) + "'";
+ }
}
private static void buildMultipleColumnRangeCondition(StringBuilder builder, TblColRef partitionDateColumn,
- TblColRef partitionTimeColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat,
- String partitionColumnTimeFormat) {
+ TblColRef partitionTimeColumn, long startInclusive, long endExclusive, PartitionDesc partitionDesc) {
String partitionDateColumnName = partitionDateColumn.getIdentity();
String partitionTimeColumnName = partitionTimeColumn.getIdentity();
+
+ String conditionDateStartValue = convertDateConditionValue(startInclusive, partitionDesc);
+ String conditionDateEndValue = convertDateConditionValue(endExclusive, partitionDesc);
+
+ String conditionTimeStartValue = convertTimeConditionValue(startInclusive, partitionDesc);
+ String conditionTimeEndValue = convertTimeConditionValue(endExclusive, partitionDesc);
+
builder.append("(");
builder.append("(");
- builder.append(partitionDateColumnName + " = '"
- + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat) + "'").append(" AND ")
- .append(partitionTimeColumnName + " >= '"
- + DateFormat.formatToDateStr(startInclusive, partitionColumnTimeFormat) + "'");
+ builder.append(partitionDateColumnName + " = " + conditionDateStartValue).append(" AND ")
+ .append(partitionTimeColumnName + " >= " + conditionTimeStartValue);
builder.append(")");
builder.append(" OR ");
builder.append("(");
- builder.append(partitionDateColumnName + " > '"
- + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat) + "'");
+ builder.append(partitionDateColumnName + " > " + conditionDateStartValue);
builder.append(")");
builder.append(")");
+
builder.append(" AND ");
builder.append("(");
builder.append("(");
- builder.append(partitionDateColumnName + " = '"
- + DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat) + "'").append(" AND ")
- .append(partitionTimeColumnName + " < '"
- + DateFormat.formatToDateStr(endExclusive, partitionColumnTimeFormat) + "'");
+ builder.append(partitionDateColumnName + " = " + conditionDateEndValue).append(" AND ")
+ .append(partitionTimeColumnName + " < " + conditionTimeEndValue);
builder.append(")");
builder.append(" OR ");
builder.append("(");
- builder.append(partitionDateColumnName + " < '"
- + DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat) + "'");
+ builder.append(partitionDateColumnName + " < " + conditionDateEndValue);
builder.append(")");
builder.append(")");
}
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index b536e29191..403c2f85c1 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -51,47 +51,139 @@ public void testDatePartition() {
TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string");
partitionDesc.setPartitionDateColumnRef(col);
partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23 21:32:29"));
+ String condition;
+
partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23"));
- String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
- Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'",
- condition);
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd HH:mm");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22 00:00' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23 21:32'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHH");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016022200' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016022321'", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHHmm");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '201602220000' AND UNKNOWN_ALIAS.DATE_COLUMN < '201602232132'", condition);
range = new TSRange(0L, 0L);
condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("1=0", condition);
}
+ @Test
+ public void testDatePartitionYmdInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "integer");
+ partitionDesc.setPartitionDateColumnRef(col);
+ partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23 20:32:29"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 20160222 AND UNKNOWN_ALIAS.DATE_COLUMN < 20160223", condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMddHHmmss");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 20160222000000 AND UNKNOWN_ALIAS.DATE_COLUMN < 20160223203229", condition);
+ }
+
+ @Test
+ public void testDatePartitionTimeMillis() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "bigint");
+ partitionDesc.setPartitionDateColumnRef(col);
+ partitionDesc.setPartitionDateColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23 20:32:29"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd HH:mm:ss");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= 1456099200000 AND UNKNOWN_ALIAS.DATE_COLUMN < 1456259549000", condition);
+ }
+
@Test
public void testTimePartition() {
PartitionDesc partitionDesc = new PartitionDesc();
TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string");
partitionDesc.setPartitionTimeColumnRef(col);
partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:12:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"));
+ String condition;
+
partitionDesc.setPartitionTimeFormat("HH");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"),
- DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition);
+
+ partitionDesc.setPartitionTimeFormat("HH:mm");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00:12' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01:00'", condition);
}
@Test
- public void testDateAndTimePartition() {
+ public void testTimePartitionIsInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "integer");
+ partitionDesc.setPartitionTimeColumnRef(col);
+ partitionDesc.setPartitionTimeColumn(col.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 19:12:00"), DateFormat.stringToMillis("2016-02-23 20:00:00"));
+ String condition;
+
+ partitionDesc.setPartitionTimeFormat("HH");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= 19 AND UNKNOWN_ALIAS.HOUR_COLUMN < 20", condition);
+ }
+
+ @Test
+ public void testDateAndTimePartitionString() {
PartitionDesc partitionDesc = new PartitionDesc();
TblColRef col1 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string");
partitionDesc.setPartitionDateColumnRef(col1);
partitionDesc.setPartitionDateColumn(col1.getCanonicalName());
- partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
TblColRef col2 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string");
partitionDesc.setPartitionTimeColumnRef(col2);
partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 12:23:00"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
partitionDesc.setPartitionTimeFormat("H");
- TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"),
- DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals(
+ "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '12') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))",
+ condition);
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ partitionDesc.setPartitionTimeFormat("HH:mm");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals(
- "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))",
+ "((UNKNOWN_ALIAS.DATE_COLUMN = '20160222' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '00:00') OR (UNKNOWN_ALIAS.DATE_COLUMN > '20160222')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '20160223' AND UNKNOWN_ALIAS.HOUR_COLUMN < '12:23') OR (UNKNOWN_ALIAS.DATE_COLUMN < '20160223'))",
condition);
}
+ @Test
+ public void testDateAndTimePartitionInt() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ TblColRef col1 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "integer");
+ partitionDesc.setPartitionDateColumnRef(col1);
+ partitionDesc.setPartitionDateColumn(col1.getCanonicalName());
+ TblColRef col2 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "bigint");
+ partitionDesc.setPartitionTimeColumnRef(col2);
+ partitionDesc.setPartitionTimeColumn(col2.getCanonicalName());
+ TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 12:23:00"));
+ String condition;
+
+ partitionDesc.setPartitionDateFormat("yyyyMMdd");
+ partitionDesc.setPartitionTimeFormat("HH");
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
+ Assert.assertEquals(
+ "((UNKNOWN_ALIAS.DATE_COLUMN = 20160222 AND UNKNOWN_ALIAS.HOUR_COLUMN >= 00) OR (UNKNOWN_ALIAS.DATE_COLUMN > 20160222)) AND ((UNKNOWN_ALIAS.DATE_COLUMN = 20160223 AND UNKNOWN_ALIAS.HOUR_COLUMN < 12) OR (UNKNOWN_ALIAS.DATE_COLUMN < 20160223))",
+ condition);
+
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services