You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jc...@apache.org on 2017/06/19 14:33:48 UTC
[1/2] calcite git commit: [CALCITE-1805] Druid adapter incorrectly
pushes down "COUNT(c)"; Druid only supports "COUNT(*)"
Repository: calcite
Updated Branches:
refs/heads/master 0c0199cf5 -> e162df1cf
[CALCITE-1805] Druid adapter incorrectly pushes down "COUNT(c)"; Druid only supports "COUNT(*)"
Close apache/calcite#463
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/12e020e0
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/12e020e0
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/12e020e0
Branch: refs/heads/master
Commit: 12e020e0f6e157f9339ecaedeb1570cc46d7f99d
Parents: 0c0199c
Author: Slim Bouguerra <sl...@gmail.com>
Authored: Wed May 31 08:25:53 2017 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Jun 19 15:32:36 2017 +0100
----------------------------------------------------------------------
.../calcite/adapter/druid/DruidRules.java | 1 +
.../org/apache/calcite/test/DruidAdapterIT.java | 114 ++++++++++---------
2 files changed, 62 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/12e020e0/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
index a0418db..8b108d5 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
@@ -122,6 +122,7 @@ public class DruidRules {
for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
switch (aggregateCall.getAggregation().getKind()) {
case COUNT:
+ return !aggregateCall.getArgList().isEmpty();
case SUM:
case SUM0:
case MIN:
http://git-wip-us.apache.org/repos/asf/calcite/blob/12e020e0/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index 5f043a2..0cd7a4b 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -1089,15 +1089,10 @@ public class DruidAdapterIT {
+ " count(\"store_sqft\") as c\n"
+ "from \"foodmart\"\n"
+ "group by floor(\"timestamp\" to MONTH)";
- String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart',"
- + "'descending':false,'granularity':'month',"
- + "'aggregations':[{'type':'longSum','name':'S','fieldName':'unit_sales'},"
- + "{'type':'count','name':'C','fieldName':'store_sqft'}],"
- + "'intervals':['1900-01-09T00:00:00.000/2992-01-10T00:00:00.000'],"
- + "'context':{'skipEmptyBuckets':true}}";
+ String druidQuery = "{'queryType':'select','dataSource':'foodmart'";
sql(sql)
.limit(3)
- .returnsUnordered("S=20957; C=6844", "S=21628; C=7033", "S=23706; C=7710")
+ .returnsUnordered("S=21081; C=5793", "S=23763; C=6762", "S=25270; C=7026")
.queryContains(druidChecker(druidQuery));
}
@@ -1112,23 +1107,24 @@ public class DruidAdapterIT {
+ "group by floor(\"timestamp\" to MONTH)\n"
+ "order by floor(\"timestamp\" to MONTH) ASC";
final String explain = "PLAN=EnumerableInterpreter\n"
- + " DruidQuery(table=[[foodmart, foodmart]], "
- + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], projects=[[FLOOR($0, "
- + "FLAG(MONTH)), $89, $71]], groups=[{0}], aggs=[[SUM($1), COUNT($2)]], sort0=[0], "
- + "dir0=[ASC])";
+ + " BindableSort(sort0=[$0], dir0=[ASC])\n"
+ + " BindableAggregate(group=[{0}], S=[SUM($1)], C=[COUNT($2)])\n"
+ + " BindableProject(M=[FLOOR($0, FLAG(MONTH))], unit_sales=[$2], store_sqft=[$1])\n"
+ + " DruidQuery(table=[[foodmart, foodmart]], "
+ + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], projects=[[$0, $71, $89]])";
sql(sql)
- .returnsOrdered("M=1997-01-01 00:00:00; S=21628; C=7033",
- "M=1997-02-01 00:00:00; S=20957; C=6844",
- "M=1997-03-01 00:00:00; S=23706; C=7710",
- "M=1997-04-01 00:00:00; S=20179; C=6588",
- "M=1997-05-01 00:00:00; S=21081; C=6865",
- "M=1997-06-01 00:00:00; S=21350; C=6912",
- "M=1997-07-01 00:00:00; S=23763; C=7752",
- "M=1997-08-01 00:00:00; S=21697; C=7038",
- "M=1997-09-01 00:00:00; S=20388; C=6662",
- "M=1997-10-01 00:00:00; S=19958; C=6478",
- "M=1997-11-01 00:00:00; S=25270; C=8231",
- "M=1997-12-01 00:00:00; S=26796; C=8716")
+ .returnsOrdered("M=1997-01-01 00:00:00; S=21628; C=5957",
+ "M=1997-02-01 00:00:00; S=20957; C=5842",
+ "M=1997-03-01 00:00:00; S=23706; C=6528",
+ "M=1997-04-01 00:00:00; S=20179; C=5523",
+ "M=1997-05-01 00:00:00; S=21081; C=5793",
+ "M=1997-06-01 00:00:00; S=21350; C=5863",
+ "M=1997-07-01 00:00:00; S=23763; C=6762",
+ "M=1997-08-01 00:00:00; S=21697; C=5915",
+ "M=1997-09-01 00:00:00; S=20388; C=5591",
+ "M=1997-10-01 00:00:00; S=19958; C=5606",
+ "M=1997-11-01 00:00:00; S=25270; C=7026",
+ "M=1997-12-01 00:00:00; S=26796; C=7338")
.explainContains(explain);
}
@@ -1139,16 +1135,16 @@ public class DruidAdapterIT {
+ "from \"foodmart\"\n"
+ "group by floor(\"timestamp\" to MONTH)\n"
+ "order by floor(\"timestamp\" to MONTH) limit 3";
- final String explain = "PLAN=EnumerableLimit(fetch=[3])\n"
- + " EnumerableInterpreter\n"
- + " DruidQuery(table=[[foodmart, foodmart]], "
- + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], projects=[[FLOOR($0, "
- + "FLAG(MONTH)), $89, $71]], groups=[{0}], aggs=[[SUM($1), COUNT($2)]], sort0=[0], "
- + "dir0=[ASC])";
+ final String explain = "PLAN=EnumerableInterpreter\n"
+ + " BindableSort(sort0=[$0], dir0=[ASC], fetch=[3])\n"
+ + " BindableAggregate(group=[{0}], S=[SUM($1)], C=[COUNT($2)])\n"
+ + " BindableProject(M=[FLOOR($0, FLAG(MONTH))], unit_sales=[$2], store_sqft=[$1])\n"
+ + " DruidQuery(table=[[foodmart, foodmart]], "
+ + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], projects=[[$0, $71, $89]])";
sql(sql)
- .returnsOrdered("M=1997-01-01 00:00:00; S=21628; C=7033",
- "M=1997-02-01 00:00:00; S=20957; C=6844",
- "M=1997-03-01 00:00:00; S=23706; C=7710")
+ .returnsOrdered("M=1997-01-01 00:00:00; S=21628; C=5957",
+ "M=1997-02-01 00:00:00; S=20957; C=5842",
+ "M=1997-03-01 00:00:00; S=23706; C=6528")
.explainContains(explain);
}
@@ -1157,15 +1153,10 @@ public class DruidAdapterIT {
+ " count(\"store_sqft\") as c\n"
+ "from \"foodmart\"\n"
+ "group by floor(\"timestamp\" to DAY)";
- String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart',"
- + "'descending':false,'granularity':'day',"
- + "'aggregations':[{'type':'longSum','name':'S','fieldName':'unit_sales'},"
- + "{'type':'count','name':'C','fieldName':'store_sqft'}],"
- + "'intervals':['1900-01-09T00:00:00.000/2992-01-10T00:00:00.000'],"
- + "'context':{'skipEmptyBuckets':true}}";
+ String druidQuery = "{'queryType':'select','dataSource':'foodmart'";
sql(sql)
.limit(3)
- .returnsUnordered("S=348; C=117", "S=589; C=189", "S=635; C=206")
+ .returnsUnordered("S=1244; C=391", "S=550; C=112", "S=580; C=171")
.queryContains(druidChecker(druidQuery));
}
@@ -1176,15 +1167,10 @@ public class DruidAdapterIT {
+ "where \"timestamp\" >= '1996-01-01 00:00:00' and "
+ " \"timestamp\" < '1998-01-01 00:00:00'\n"
+ "group by floor(\"timestamp\" to MONTH)";
- String druidQuery = "{'queryType':'timeseries','dataSource':'foodmart',"
- + "'descending':false,'granularity':'month',"
- + "'aggregations':[{'type':'longSum','name':'S','fieldName':'unit_sales'},"
- + "{'type':'count','name':'C','fieldName':'store_sqft'}],"
- + "'intervals':['1996-01-01T00:00:00.000/1998-01-01T00:00:00.000'],"
- + "'context':{'skipEmptyBuckets':true}}";
+ String druidQuery = "{'queryType':'select','dataSource':'foodmart'";
sql(sql)
.limit(3)
- .returnsUnordered("S=20957; C=6844", "S=21628; C=7033", "S=23706; C=7710")
+ .returnsUnordered("S=21081; C=5793", "S=23763; C=6762", "S=25270; C=7026")
.queryContains(druidChecker(druidQuery));
}
@@ -1292,16 +1278,18 @@ public class DruidAdapterIT {
+ "from \"foodmart\"\n"
+ "group by \"state_province\"\n"
+ "order by 2 desc limit 2";
- final String explain = "PLAN="
- + "EnumerableInterpreter\n"
+ final String explain = "PLAN=EnumerableInterpreter\n"
+ " BindableSort(sort0=[$1], dir0=[DESC], fetch=[2])\n"
+ " BindableProject(state_province=[$0], CDC=[FLOOR($1)])\n"
- + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], groups=[{30}], aggs=[[COUNT(DISTINCT $29)]])\n";
+ + " BindableAggregate(group=[{1}], agg#0=[COUNT($0)])\n"
+ + " DruidQuery(table=[[foodmart, foodmart]], "
+ + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], groups=[{29, 30}], "
+ + "aggs=[[]])";
final String druidQuery = "{'queryType':'groupBy','dataSource':'foodmart',"
- + "'granularity':'all',"
- + "'dimensions':[{'type':'default','dimension':'state_province'}],"
- + "'limitSpec':{'type':'default'},"
- + "'aggregations':[{'type':'cardinality','name':'$f1','fieldNames':['city']}],"
+ + "'granularity':'all','dimensions':[{'type':'default','dimension':'city'},"
+ + "{'type':'default','dimension':'state_province'}],"
+ + "'limitSpec':{'type':'default'},'aggregations':[{'type':'longSum',"
+ + "'name':'dummy_agg','fieldName':'dummy_agg'}],"
+ "'intervals':['1900-01-09T00:00:00.000/2992-01-10T00:00:00.000']}";
sql(sql)
.explainContains(explain)
@@ -2174,6 +2162,26 @@ public class DruidAdapterIT {
.queryContains(druidChecker("'queryType':'timeseries'"));
}
+
+ /**
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-1805">[CALCITE-1805]
+ * Druid adapter can not handel count column without adding support for nested queries</a>.
+ */
+ @Test public void testCountColumn() {
+
+ final String sql = "SELECT count(\"countryName\") FROM (SELECT \"countryName\" FROM "
+ + "\"wikiticker\" WHERE \"countryName\" IS NOT NULL) as a";
+ sql(sql, WIKI_AUTO2).returnsUnordered("EXPR$0=3799");
+
+ final String sql2 = "SELECT count(\"countryName\") FROM (SELECT \"countryName\" FROM "
+ + "\"wikiticker\") as a";
+ sql(sql2, WIKI_AUTO2).returnsUnordered("EXPR$0=3799").explainContains("PLAN"
+ + "=EnumerableInterpreter\n"
+ + " BindableAggregate(group=[{}], EXPR$0=[COUNT($0)])\n"
+ + " DruidQuery(table=[[wiki, wikiticker]], "
+ + "intervals=[[1900-01-01T00:00:00.000/3000-01-01T00:00:00.000]], projects=[[$7]])");
+ }
+
}
// End DruidAdapterIT.java
[2/2] calcite git commit: [CALCITE-1805] Druid adapter incorrectly
pushes down "COUNT(c)"; Druid only supports "COUNT(*)"
Posted by jc...@apache.org.
[CALCITE-1805] Druid adapter incorrectly pushes down "COUNT(c)"; Druid only supports "COUNT(*)"
Fixes DruidAdapterIT.testGroupByAvgSumCount and DruidAdapterIT.testTimeExtractThatCannotBePushed
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/e162df1c
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/e162df1c
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/e162df1c
Branch: refs/heads/master
Commit: e162df1cfeb615da909ac649998e81f3aa3faf20
Parents: 12e020e
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Jun 19 15:29:11 2017 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Jun 19 15:33:12 2017 +0100
----------------------------------------------------------------------
.../calcite/interpreter/AggregateNode.java | 90 ++++++++++++++++++--
.../org/apache/calcite/test/DruidAdapterIT.java | 15 +---
2 files changed, 88 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/e162df1c/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
index 5a5d265..6a54b87 100644
--- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
+++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java
@@ -136,6 +136,24 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
}
return new UdaAccumulatorFactory(
AggregateFunctionImpl.create(clazz), call);
+ } else if (call.getAggregation() == SqlStdOperatorTable.SUM0) {
+ final Class<?> clazz;
+ switch (call.type.getSqlTypeName()) {
+ case DOUBLE:
+ case REAL:
+ case FLOAT:
+ clazz = DoubleSum0.class;
+ break;
+ case INTEGER:
+ clazz = IntSum0.class;
+ break;
+ case BIGINT:
+ default:
+ clazz = LongSum0.class;
+ break;
+ }
+ return new UdaAccumulatorFactory(
+ AggregateFunctionImpl.create(clazz), call);
} else {
final JavaTypeFactory typeFactory =
(JavaTypeFactory) rel.getCluster().getTypeFactory();
@@ -379,8 +397,8 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
public static class IntSum {
public IntSum() {
}
- public int init() {
- return 0;
+ public Integer init() {
+ return null;
}
public int add(int accumulator, int v) {
return accumulator + v;
@@ -388,7 +406,7 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
public int merge(int accumulator0, int accumulator1) {
return accumulator0 + accumulator1;
}
- public int result(int accumulator) {
+ public Integer result(Integer accumulator) {
return accumulator;
}
}
@@ -398,8 +416,8 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
public static class LongSum {
public LongSum() {
}
- public long init() {
- return 0L;
+ public Long init() {
+ return null;
}
public long add(long accumulator, long v) {
return accumulator + v;
@@ -407,7 +425,7 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
public long merge(long accumulator0, long accumulator1) {
return accumulator0 + accumulator1;
}
- public long result(long accumulator) {
+ public Long result(Long accumulator) {
return accumulator;
}
}
@@ -417,6 +435,63 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
public static class DoubleSum {
public DoubleSum() {
}
+ public Double init() {
+ return null;
+ }
+ public double add(double accumulator, double v) {
+ return accumulator + v;
+ }
+ public double merge(double accumulator0, double accumulator1) {
+ return accumulator0 + accumulator1;
+ }
+ public Double result(Double accumulator) {
+ return accumulator;
+ }
+ }
+
+ /** Implementation of {@code SUM0} over INTEGER values as a user-defined
+ * aggregate. */
+ public static class IntSum0 {
+ public IntSum0() {
+ }
+ public int init() {
+ return 0;
+ }
+ public int add(int accumulator, int v) {
+ return accumulator + v;
+ }
+ public int merge(int accumulator0, int accumulator1) {
+ return accumulator0 + accumulator1;
+ }
+ public int result(int accumulator) {
+ return accumulator;
+ }
+ }
+
+ /** Implementation of {@code SUM0} over BIGINT values as a user-defined
+ * aggregate. */
+ public static class LongSum0 {
+ public LongSum0() {
+ }
+ public long init() {
+ return 0L;
+ }
+ public long add(long accumulator, long v) {
+ return accumulator + v;
+ }
+ public long merge(long accumulator0, long accumulator1) {
+ return accumulator0 + accumulator1;
+ }
+ public long result(long accumulator) {
+ return accumulator;
+ }
+ }
+
+ /** Implementation of {@code SUM0} over DOUBLE values as a user-defined
+ * aggregate. */
+ public static class DoubleSum0 {
+ public DoubleSum0() {
+ }
public double init() {
return 0D;
}
@@ -480,6 +555,9 @@ public class AggregateNode extends AbstractSingleNode<Aggregate> {
public void send(Row row) {
final Object[] args = {value, row.getValues()[factory.argOrdinal]};
+ if (args[0] == null) {
+ args[0] = 0; // first value, set to zero
+ }
for (int i = 1; i < args.length; i++) {
if (args[i] == null) {
return; // one of the arguments is null; don't add to the total
http://git-wip-us.apache.org/repos/asf/calcite/blob/e162df1c/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index 0cd7a4b..719d3bd 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -1071,17 +1071,11 @@ public class DruidAdapterIT {
+ "from \"foodmart\"\n"
+ "group by \"state_province\"\n"
+ "order by 1";
- String druidQuery = "'aggregations':["
- + "{'type':'longSum','name':'$f1','fieldName':'unit_sales'},"
- + "{'type':'count','name':'$f2','fieldName':'unit_sales'},"
- + "{'type':'count','name':'C','fieldName':'store_sqft'},"
- + "{'type':'count','name':'C0'}],"
- + "'intervals':['1900-01-09T00:00:00.000/2992-01-10T00:00:00.000']}";
sql(sql)
.limit(2)
- .returnsUnordered("state_province=CA; A=3; S=74748; C=24441; C0=24441",
+ .returnsUnordered("state_province=CA; A=3; S=74748; C=16347; C0=24441",
"state_province=OR; A=3; S=67659; C=21610; C0=21610")
- .queryContains(druidChecker(druidQuery));
+ .queryContains(druidChecker("'queryType':'select'"));
}
@Test public void testGroupByMonthGranularity() {
@@ -2075,13 +2069,12 @@ public class DruidAdapterIT {
+ "\"product_id\" = 1558 group by extract(CENTURY from \"timestamp\")";
final String plan = "PLAN=EnumerableInterpreter\n"
+ " BindableAggregate(group=[{0}])\n"
- + " BindableProject(EXPR$0=[/INT(EXTRACT_DATE(FLAG(YEAR), /INT(Reinterpret($0), "
- + "86400000)), 100)])\n"
+ + " BindableProject(EXPR$0=[EXTRACT_DATE(FLAG(CENTURY), /INT(Reinterpret($0), 86400000))])\n"
+ " DruidQuery(table=[[foodmart, foodmart]], "
+ "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], filter=[=($1, 1558)], "
+ "projects=[[$0]])";
sql(sql).explainContains(plan).queryContains(druidChecker("'queryType':'select'"))
- .returnsUnordered("EXPR$0=19");
+ .returnsUnordered("EXPR$0=20");
}
/** Test case for