You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jc...@apache.org on 2018/04/03 01:56:17 UTC
[1/2] calcite git commit: [CALCITE-2222] Add Quarter timeunit as a
valid unit to pushdown to Druid
Repository: calcite
Updated Branches:
refs/heads/master a2aaf3924 -> 6b35306e7
[CALCITE-2222] Add Quarter timeunit as a valid unit to pushdown to Druid
Close apache/calcite#652
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/09693abb
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/09693abb
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/09693abb
Branch: refs/heads/master
Commit: 09693abbe333ecf3c973ddee33ff8d31b20386d4
Parents: a2aaf39
Author: Slim <sl...@gmail.com>
Authored: Thu Mar 22 22:03:21 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Apr 2 18:29:17 2018 -0700
----------------------------------------------------------------------
.../adapter/druid/TimeExtractionFunction.java | 1 +
.../org/apache/calcite/test/DruidAdapterIT.java | 73 ++++++++++++++++++++
2 files changed, 74 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/09693abb/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java b/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
index 5b0265e..552d221 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/TimeExtractionFunction.java
@@ -59,6 +59,7 @@ public class TimeExtractionFunction implements ExtractionFunction {
private static final ImmutableSet<TimeUnitRange> VALID_TIME_FLOOR = Sets.immutableEnumSet(
TimeUnitRange.YEAR,
+ TimeUnitRange.QUARTER,
TimeUnitRange.MONTH,
TimeUnitRange.DAY,
TimeUnitRange.WEEK,
http://git-wip-us.apache.org/repos/asf/calcite/blob/09693abb/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index 7c7ba4a..0a9bc03 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -4413,6 +4413,79 @@ public class DruidAdapterIT {
+ "{'type':'filter','filter':{'type':'bound','dimension':'$f2','lower':'0',"
+ "'lowerStrict':true,'ordering':'numeric'}}}"));
}
+
+ @Test
+ public void testFloorQuarter() {
+ String sql = "SELECT floor(\"timestamp\" TO quarter), SUM(\"store_sales\") FROM "
+ + FOODMART_TABLE
+ + " GROUP BY floor(\"timestamp\" TO quarter)";
+
+ sql(sql, FOODMART).queryContains(
+ druidChecker(
+ "{\"queryType\":\"timeseries\",\"dataSource\":\"foodmart\",\"descending\":false,"
+ + "\"granularity\":{\"type\":\"period\",\"period\":\"P3M\",\"timeZone\":\"UTC\"},"
+ + "\"aggregations\":[{\"type\":\"doubleSum\",\"name\":\"EXPR$1\",\"fieldName\":\"store_sales\"}],"
+ + "\"intervals\":[\"1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z\"],\"context\":{\"skipEmptyBuckets\":true}}"));
+ }
+
+ @Test
+ public void testFloorQuarterPlusDim() {
+ String sql =
+ "SELECT floor(\"timestamp\" TO quarter),\"product_id\", SUM(\"store_sales\") FROM "
+ + FOODMART_TABLE
+ + " GROUP BY floor(\"timestamp\" TO quarter), \"product_id\"";
+
+ sql(sql, FOODMART).queryContains(
+ druidChecker(
+ "{\"queryType\":\"groupBy\",\"dataSource\":\"foodmart\",\"granularity\":\"all\",\"dimensions\":"
+ + "[{\"type\":\"extraction\",\"dimension\":\"__time\",\"outputName\":\"floor_quarter\",\"extractionFn\":{\"type\":\"timeFormat\"",
+ "\"granularity\":{\"type\":\"period\",\"period\":\"P3M\",\"timeZone\":\"UTC\"},\"timeZone\":\"UTC\",\"locale\":\"und\"}},"
+ + "{\"type\":\"default\",\"dimension\":\"product_id\",\"outputName\":\"product_id\",\"outputType\":\"STRING\"}],"
+ + "\"limitSpec\":{\"type\":\"default\"},\"aggregations\":[{\"type\":\"doubleSum\",\"name\":\"EXPR$2\",\"fieldName\":\"store_sales\"}],"
+ + "\"intervals\":[\"1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z\"]}"));
+ }
+
+
+ @Test
+ public void testExtractQuarterPlusDim() {
+ String sql =
+ "SELECT EXTRACT(quarter from \"timestamp\"),\"product_id\", SUM(\"store_sales\") FROM "
+ + FOODMART_TABLE
+ + " WHERE \"product_id\" = 1"
+ + " GROUP BY EXTRACT(quarter from \"timestamp\"), \"product_id\"";
+
+ sql(sql, FOODMART).returnsOrdered("EXPR$0=1; product_id=1; EXPR$2=37.050000000000004\n"
+ + "EXPR$0=2; product_id=1; EXPR$2=62.7\n"
+ + "EXPR$0=3; product_id=1; EXPR$2=88.35\n"
+ + "EXPR$0=4; product_id=1; EXPR$2=48.45")
+ .queryContains(
+ druidChecker(
+ "{\"queryType\":\"groupBy\",\"dataSource\":\"foodmart\",\"granularity\":\"all\",\"dimensions\":"
+ + "[{\"type\":\"default\",\"dimension\":\"vc\",\"outputName\":\"vc\",\"outputType\":\"LONG\"},"
+ + "{\"type\":\"default\",\"dimension\":\"product_id\",\"outputName\":\"product_id\",\"outputType\":\"STRING\"}],"
+ + "\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"vc\",\"expression\":\"timestamp_extract(\\\"__time\\\",",
+ "QUARTER"
+ ));
+ }
+
+ @Test
+ public void testExtractQuarter() {
+ String sql = "SELECT EXTRACT(quarter from \"timestamp\"), SUM(\"store_sales\") FROM "
+ + FOODMART_TABLE
+ + " GROUP BY EXTRACT(quarter from \"timestamp\")";
+
+ sql(sql, FOODMART).returnsOrdered("EXPR$0=1; EXPR$1=139628.34999999971\n"
+ + "EXPR$0=2; EXPR$1=132666.26999999944\n"
+ + "EXPR$0=3; EXPR$1=140271.88999999964\n"
+ + "EXPR$0=4; EXPR$1=152671.61999999985")
+ .queryContains(
+ druidChecker(
+ "{\"queryType\":\"groupBy\",\"dataSource\":\"foodmart\",\"granularity\":\"all\","
+ + "\"dimensions\":[{\"type\":\"default\",\"dimension\":\"vc\",\"outputName\":\"vc\",\"outputType\":\"LONG\"}],"
+ + "\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"vc\",\"expression\":\"timestamp_extract(\\\"__time\\\",",
+ "QUARTER"
+ ));
+ }
}
// End DruidAdapterIT.java
[2/2] calcite git commit: [CALCITE-2226] Druid adapter: Substring
operator converter does not handle non-constant literals correctly
Posted by jc...@apache.org.
[CALCITE-2226] Druid adapter: Substring operator converter does not handle non-constant literals correctly
Close apache/calcite#653
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/6b35306e
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/6b35306e
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/6b35306e
Branch: refs/heads/master
Commit: 6b35306e7818cb3e67e27b9edbb00b1ed8338cdd
Parents: 09693ab
Author: Slim <sl...@gmail.com>
Authored: Wed Mar 21 19:06:33 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Apr 2 18:32:31 2018 -0700
----------------------------------------------------------------------
.../druid/SubstringOperatorConversion.java | 40 +++++++++++++++-----
.../org/apache/calcite/test/DruidAdapterIT.java | 33 ++++++++++++++++
2 files changed, 64 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/6b35306e/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java b/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java
index d2342f3..9d66a7b 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/SubstringOperatorConversion.java
@@ -20,6 +20,7 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
@@ -43,20 +44,41 @@ public class SubstringOperatorConversion implements DruidSqlOperatorConverter {
return null;
}
- final int index = RexLiteral.intValue(call.getOperands().get(1)) - 1;
+ final String startIndex;
+ final String length;
// SQL is 1-indexed, Druid is 0-indexed.
- final int length;
+ if (!call.getOperands().get(1).isA(SqlKind.LITERAL)) {
+ final String arg1 = DruidExpressions.toDruidExpression(
+ call.getOperands().get(1), rowType, query);
+ if (arg1 == null) {
+ // can not infer start index expression bailout.
+ return null;
+ }
+ startIndex = DruidQuery.format("(%s - 1)", arg1);
+ } else {
+ startIndex = DruidExpressions.numberLiteral(
+ RexLiteral.intValue(call.getOperands().get(1)) - 1);
+ }
+
if (call.getOperands().size() > 2) {
- //case substring from index with length
- length = RexLiteral.intValue(call.getOperands().get(2));
+ //case substring from start index with length
+ if (!call.getOperands().get(2).isA(SqlKind.LITERAL)) {
+ // case it is an expression try to parse it
+ length = DruidExpressions.toDruidExpression(
+ call.getOperands().get(2), rowType, query);
+ if (length == null) {
+ return null;
+ }
+ } else {
+ // case length is a constant
+ length = DruidExpressions.numberLiteral(RexLiteral.intValue(call.getOperands().get(2)));
+ }
+
} else {
//case substring from index to the end
- length = -1;
+ length = DruidExpressions.numberLiteral(-1);
}
- return DruidQuery.format("substring(%s, %s, %s)",
- arg,
- DruidExpressions.numberLiteral(index),
- DruidExpressions.numberLiteral(length));
+ return DruidQuery.format("substring(%s, %s, %s)", arg, startIndex, length);
}
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/6b35306e/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index 0a9bc03..7815f5b 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -3888,6 +3888,39 @@ public class DruidAdapterIT {
+ "\"outputType\":\"STRING\"}]"));
}
+ @Test
+ public void testSubStringWithNonConstantIndexes() {
+ final String sql = "SELECT COUNT(*) FROM "
+ + FOODMART_TABLE
+ + " WHERE SUBSTRING(\"product_id\" from CAST(\"store_cost\" as INT)/1000 + 2 "
+ + "for CAST(\"product_id\" as INT)) like '1%'";
+
+ sql(sql, FOODMART).returnsOrdered("EXPR$0=10893")
+ .queryContains(
+ druidChecker("\"queryType\":\"timeseries\"", "like(substring(\\\"product_id\\\""))
+ .explainContains(
+ "PLAN=EnumerableInterpreter\n DruidQuery(table=[[foodmart, foodmart]], "
+ + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], "
+ + "filter=[LIKE(SUBSTRING($1, +(/(CAST($91):INTEGER, 1000), 2), CAST($1):INTEGER), '1%')], "
+ + "groups=[{}], aggs=[[COUNT()]])\n\n");
+ }
+
+ @Test
+ public void testSubStringWithNonConstantIndex() {
+ final String sql = "SELECT COUNT(*) FROM "
+ + FOODMART_TABLE
+ + " WHERE SUBSTRING(\"product_id\" from CAST(\"store_cost\" as INT)/1000 + 1) like '1%'";
+
+ sql(sql, FOODMART).returnsOrdered("EXPR$0=36839")
+ .queryContains(druidChecker("like(substring(\\\"product_id\\\""))
+ .explainContains(
+ "PLAN=EnumerableInterpreter\n DruidQuery(table=[[foodmart, foodmart]], "
+ + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], "
+ + "filter=[LIKE(SUBSTRING($1, +(/(CAST($91):INTEGER, 1000), 1)), '1%')],"
+ + " groups=[{}], aggs=[[COUNT()]])\n\n");
+ }
+
+
/**
* Test case for https://issues.apache.org/jira/browse/CALCITE-2098.
* Need to make sure that when there we have a valid filter with no conjunction we still push