You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2016/06/09 00:46:09 UTC
[1/6] calcite git commit: [CALCITE-1279] Druid "select" query gives
ClassCastException [Forced Update!]
Repository: calcite
Updated Branches:
refs/heads/branch-1.8 8e2ac9c98 -> 4987d5406 (forced update)
[CALCITE-1279] Druid "select" query gives ClassCastException
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/23c8e458
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/23c8e458
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/23c8e458
Branch: refs/heads/branch-1.8
Commit: 23c8e4588c2c23703cb90ad66e51b90a8a53d7df
Parents: a02da27
Author: Julian Hyde <jh...@apache.org>
Authored: Wed Jun 8 12:55:22 2016 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Jun 8 17:22:29 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/calcite/adapter/druid/DruidQuery.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/23c8e458/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
index bc51e88..0a9dbec 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
@@ -399,9 +399,8 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
generator.writeFieldName("pagingSpec");
generator.writeStartObject();
final int fetch =
- Integer.parseInt(
- CalciteConnectionProperty.DRUID_FETCH.wrap(new Properties())
- .getString());
+ CalciteConnectionProperty.DRUID_FETCH.wrap(new Properties())
+ .getInt();
generator.writeNumberField("threshold", fetch);
generator.writeEndObject();
[6/6] calcite git commit: Release notes
Posted by jh...@apache.org.
Release notes
Change versions from 1.7 to 1.8.
Tell apache-rat to ignore a Druid JSON file, because JSON files cannot
have headers.
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/4987d540
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/4987d540
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/4987d540
Branch: refs/heads/branch-1.8
Commit: 4987d54061783ee5c707a60140b3e367baccdd49
Parents: d18da01
Author: Julian Hyde <jh...@apache.org>
Authored: Tue May 17 15:24:27 2016 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Jun 8 17:36:58 2016 -0700
----------------------------------------------------------------------
README | 2 +-
pom.xml | 3 +-
site/_docs/adapter.md | 2 +-
site/_docs/history.md | 169 +++++++++++++++++++++++++++++++++++++++++++
site/_docs/howto.md | 6 +-
site/community/index.md | 13 ++--
6 files changed, 182 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/4987d540/README
----------------------------------------------------------------------
diff --git a/README b/README
index f07e842..b21e547 100644
--- a/README
+++ b/README
@@ -1,4 +1,4 @@
-Apache Calcite release 1.7.0
+Apache Calcite release 1.8.0
This is a source or binary distribution of Apache Calcite.
http://git-wip-us.apache.org/repos/asf/calcite/blob/4987d540/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0bcd67c..ff10873 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,7 +49,7 @@ limitations under the License.
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<top.dir>${project.basedir}</top.dir>
<version.major>1</version.major>
- <version.minor>7</version.minor>
+ <version.minor>8</version.minor>
<!-- This list is in alphabetical order. -->
<airlift-tpch.version>0.1</airlift-tpch.version>
@@ -438,6 +438,7 @@ limitations under the License.
<exclude>src/main/resources/META-INF/services/java.sql.Driver</exclude>
<exclude>**/src/test/resources/**/*.csv</exclude>
<exclude>**/src/test/resources/bug/archers.json</exclude>
+ <exclude>**/src/test/resources/foodmart-schema.spec</exclude>
<!-- Exclude all of avatica, RAT is run in Avatica -->
<exclude>avatica/**</exclude>
http://git-wip-us.apache.org/repos/asf/calcite/blob/4987d540/site/_docs/adapter.md
----------------------------------------------------------------------
diff --git a/site/_docs/adapter.md b/site/_docs/adapter.md
index 10cb8f8..987538f 100644
--- a/site/_docs/adapter.md
+++ b/site/_docs/adapter.md
@@ -64,7 +64,7 @@ where `property`, `property2` are properties as described below.
as implemented by Avatica's
<a href="{{ site.apiRoot }}/org/apache/calcite/avatica/ConnectStringParser.html">ConnectStringParser</a>.)
-JDBC connect string parameters
+## JDBC connect string parameters
| Property | Description |
|:-------- |:------------|
http://git-wip-us.apache.org/repos/asf/calcite/blob/4987d540/site/_docs/history.md
----------------------------------------------------------------------
diff --git a/site/_docs/history.md b/site/_docs/history.md
index fcd20c2..7aada37 100644
--- a/site/_docs/history.md
+++ b/site/_docs/history.md
@@ -28,6 +28,175 @@ For a full list of releases, see
Downloads are available on the
[downloads page]({{ site.baseurl }}/downloads/).
+## <a href="https://github.com/apache/calcite/releases/tag/calcite-1.8.0">1.8.0</a> / 2016-06-13
+{: #v1-8-0}
+
+This release adds adapters for
+<a href="https://issues.apache.org/jira/browse/CALCITE-1253">Elasticsearch</a> and
+<a href="https://issues.apache.org/jira/browse/CALCITE-1121">Druid</a>.
+It is also now easier to
+<a href="https://issues.apache.org/jira/browse/CALCITE-1259">make a JDBC connection based upon a single adapter</a>.
+
+There are several new SQL features: `UNNEST` with
+<a href="https://issues.apache.org/jira/browse/CALCITE-855">multiple arguments</a>,
+<a href="https://issues.apache.org/jira/browse/CALCITE-1250">MAP arguments</a>
+and <a href="https://issues.apache.org/jira/browse/CALCITE-1225">with a JOIN</a>;
+a <a href="https://issues.apache.org/jira/browse/CALCITE-1168">DESCRIBE</a> statement;
+and a <a href="https://issues.apache.org/jira/browse/CALCITE-1115">TRANSLATE</a>
+function like the one in Oracle and PostgreSQL.
+We also added support for
+<a href="https://issues.apache.org/jira/browse/CALCITE-1120">SELECT without FROM</a>
+(equivalent to the `VALUES` clause, and widely used in MySQL and PostgreSQL),
+and added a
+[conformance]({{ site.baseurl }}/docs/adapter.html#jdbc-connect-string-parameters)
+parameter to allow you to selectively enable this and other SQL features.
+
+And a couple of dozen bug-fixes and enhancements to planner rules and APIs.
+
+Compatibility: This release is tested
+on Linux, Mac OS X, Microsoft Windows;
+using Oracle JDK 1.7, 1.8;
+Guava versions 12.0.1 to 19.0;
+other software versions as specified in `pom.xml`.
+
+New features
+
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1177">CALCITE-1177</a>]
+ Extend list of supported time units in `EXTRACT`, `CEIL` and `FLOOR` functions
+ (Venki Korukanti)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1259">CALCITE-1259</a>]
+ Allow connecting to a single schema without writing a model
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-750">CALCITE-750</a>]
+ Support aggregates within windowed aggregates (Gautam Parai)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1250">CALCITE-1250</a>]
+ `UNNEST` applied to `MAP` data type (Johannes Schulte)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1253">CALCITE-1253</a>]
+ Elasticsearch adapter (Subhobrata Dey)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1228">CALCITE-1228</a>]
+ Bind parameters in `INSERT`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1120">CALCITE-1120</a>]
+ `SELECT` without `FROM` (Jimmy Xiang)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-855">CALCITE-855</a>]
+ `UNNEST` with multiple arguments
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1225">CALCITE-1225</a>]
+ `UNNEST` with `JOIN`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1115">CALCITE-1115</a>]
+ Add `TRANSLATE` function with 3 parameters, like the one in Oracle (Javanshir
+ Yelchiyev)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1168">CALCITE-1168</a>]
+ Add `DESCRIBE` statement (Arina Ielchiieva)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1121">CALCITE-1121</a>]
+ Druid adapter
+ * [<a href="https://issues.apache.org/jira/browse/CALCITE-1276">CALCITE-1276</a>]
+ In Druid adapter, deduce tables and columns if not specified
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1207">CALCITE-1207</a>]
+ Allow numeric connection properties, and 'K', 'M', 'G' suffixes
+
+Planner rules
+
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1235">CALCITE-1235</a>]
+ Fully push down `LIMIT` + `OFFSET` in Cassandra
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1216">CALCITE-1216</a>]
+ Rule to convert `Filter`-on-`Scan` to materialized view (Amogh Margoor)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1200">CALCITE-1200</a>]
+ Extend `RelOptUtil.splitJoinCondition` to handle `IS NOT DISTINCT FROM`
+ (Venki Korukanti)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1211">CALCITE-1211</a>]
+ Allow free use of `CassandraSort` for `LIMIT`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1210">CALCITE-1210</a>]
+ Allow UUID filtering in Cassandra
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1182">CALCITE-1182</a>]
+ Add `ProjectRemoveRule` to pre-processing program of materialization
+ substitution
+
+Bug fixes, API changes and minor enhancements
+
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1281">CALCITE-1281</a>]
+ Druid adapter wrongly returns all numeric values as `int` or `float`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1279">CALCITE-1279</a>]
+ Druid "select" query gives `ClassCastException`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1277">CALCITE-1277</a>]
+ Rat fails on source distribution due to `git.properties`
+* Update KEYS
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1252">CALCITE-1252</a>]
+ Handle `ANY` type in `RexBuilder.ensureType` and `TypeFactory.leastRestrictive`
+ (Mehand Baid, Minji Kim)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1151">CALCITE-1151</a>]
+ Fix `SqlSetOption` to correctly handle `SqlOperator.createCall`
+ (Sudheesh Katkam, Minji Kim)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1106">CALCITE-1106</a>]
+ Expose constructor for `ProjectJoinTransposeRule` (Minji Kim)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1107">CALCITE-1107</a>]
+ Make `SqlSumEmptyIsZeroAggFunction` constructor public (Minji Kim)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1269">CALCITE-1269</a>]
+ Replace `IntList` with Guava `Ints` class
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1239">CALCITE-1239</a>]
+ Upgrade to avatica-1.8.0
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1266">CALCITE-1266</a>]
+ `RelBuilder.field` gets offsets wrong
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1264">CALCITE-1264</a>]
+ `Litmus` argument interpolation (Chris Baynes)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1245">CALCITE-1245</a>]
+ Allow `RelBuilder.scan` to take qualified table name (Chris Baynes)
+* Move code from `Enumerables` to `EnumerableDefaults`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1246">CALCITE-1246</a>]
+ Cleanup duplicate variables in `JoinPushThroughJoinRule` (Yi Xinglu)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1241">CALCITE-1241</a>]
+ Add a Freemarker variable for adding non reserved keyword list to `Parser.jj`
+ template (Venki Korukanti)
+ * Create a table in `SqlParserTest` of reserved keywords from various versions
+ of the SQL standard
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1150">CALCITE-1150</a>]
+ Add dynamic record type and dynamic star for schema-on-read table
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1238">CALCITE-1238</a>]
+ Unparsing a query with `LIMIT` but no `ORDER BY` gives invalid SQL (Emmanuel
+ Bastien)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1230">CALCITE-1230</a>]
+ Add SQLSTATE reference data as `enum SqlState` in Avatica, and
+ deprecate `SqlStateCodes` in Calcite
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1199">CALCITE-1199</a>]
+ Incorrect trimming of `CHAR` when performing cast to `VARCHAR`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1219">CALCITE-1219</a>]
+ Add method `SqlOperatorBinding.isOperandLiteral()` (Hsuan-Yi Chu)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1222">CALCITE-1222</a>]
+ `DatabaseMetaData.getColumnLabel` returns null when query has `ORDER BY`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1215">CALCITE-1215</a>]
+ Fix missing override in `CassandraTable`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1212">CALCITE-1212</a>]
+ Fix NPE on some Cassandra projects
+* Remove trailing spaces from all source files
+* Move HTTP utilities from Splunk adapter to core
+* Test case for
+ [<a href="https://issues.apache.org/jira/browse/PHOENIX-2767">PHOENIX-2767</a>],
+ non-constant in `IN`
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1166">CALCITE-1166</a>]
+ Disallow sub-classes of `RelOptRuleOperand`
+* Remove all calls to deprecated methods
+* Add class `ConsList`
+* More of [<a href="https://issues.apache.org/jira/browse/CALCITE-999">CALCITE-999</a>]
+ Clean up maven POM files
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1204">CALCITE-1204</a>]
+ Fix invalid Javadoc and suppress checkstyle "errors"
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1170">CALCITE-1170</a>]
+ Allow `SqlSetOperator` to be overridden, as a regular `SqlOperator` can
+ (Hsuan-Yi Chu)
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-746">CALCITE-746</a>]
+ Allow apache-rat to be run outside of release process
+
+Web site and documentation
+
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1273">CALCITE-1273</a>]
+ Following
+ [<a href="https://issues.apache.org/jira/browse/CALCITE-306">CALCITE-306</a>],
+ update references to `EnumerableTableAccessRel` to `EnumerableTableScan`
+* Fix typo in SQL (Yi Xinglu)
+* Site: add committer, and post slides/video from Polyalgebra talk
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1203">CALCITE-1203</a>]
+ Update to github-pages-67
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-1202">CALCITE-1202</a>]
+ Lock version of Jekyll for bundler
+* Site: add upcoming talks, and fix link to Apache phonebook
+
## <a href="https://github.com/apache/calcite/releases/tag/calcite-1.7.0">1.7.0</a> / 2016-03-22
{: #v1-7-0}
http://git-wip-us.apache.org/repos/asf/calcite/blob/4987d540/site/_docs/howto.md
----------------------------------------------------------------------
diff --git a/site/_docs/howto.md b/site/_docs/howto.md
index 723b39c..b31d985 100644
--- a/site/_docs/howto.md
+++ b/site/_docs/howto.md
@@ -39,8 +39,8 @@ Unpack the source distribution `.tar.gz` or `.zip` file,
then build using maven:
{% highlight bash %}
-$ tar xvfz calcite-1.7.0-source.tar.gz
-$ cd calcite-1.7.0
+$ tar xvfz calcite-1.8.0-source.tar.gz
+$ cd calcite-1.8.0
$ mvn install
{% endhighlight %}
@@ -113,7 +113,7 @@ environment, as follows.
## Running integration tests
For testing Calcite's external adapters, a test virtual machine should be used.
-The VM includes H2, HSQLDB, MySQL, MongoDB, and PostgreSQL.
+The VM includes Cassandra, Druid, H2, HSQLDB, MySQL, MongoDB, and PostgreSQL.
Test VM requires 5GiB of disk space and it takes 30 minutes to build.
http://git-wip-us.apache.org/repos/asf/calcite/blob/4987d540/site/community/index.md
----------------------------------------------------------------------
diff --git a/site/community/index.md b/site/community/index.md
index 7e8bec0..bfbbd96 100644
--- a/site/community/index.md
+++ b/site/community/index.md
@@ -26,8 +26,6 @@ limitations under the License.
# Upcoming talks
-* 2016/05/10 <a href="http://events.linuxfoundation.org/events/apache-big-data-north-america/program/schedule">ApacheCon Big Data North America</a>, Vancouver
-* 2016/05/24 <a href="http://www-conf.slac.stanford.edu/xldb2016/Program.asp">XLDB</a>, Palo Alto
* 2016/06/29 <a href="http://hadoopsummit.org/san-jose/agenda/">Hadoop Summit</a>, San Jose
# Project Members
@@ -39,7 +37,7 @@ Name (Apache ID) | Github | Org | Role
# Mailing Lists
-There are several development mailing lists for Calcite:
+There are several mailing lists for Calcite:
* [dev@calcite.apache.org](mailto:dev@calcite.apache.org) - Development discussions
[[archive](https://mail-archives.apache.org/mod_mbox/calcite-dev/)]
@@ -49,8 +47,8 @@ There are several development mailing lists for Calcite:
[[archive](https://mail-archives.apache.org/mod_mbox/calcite-commits/)]
You can subscribe to the lists by sending email to
-*list*-subscribe@calcite.apache.org and unsubscribe by sending email to
-*list*-unsubscribe@calcite.apache.org.
+*{list}*-subscribe@calcite.apache.org and unsubscribe by sending email to
+*{list}*-unsubscribe@calcite.apache.org (where *{list}* is either "dev", "issues", or "commits").
# Help
@@ -66,9 +64,9 @@ Need help with Calcite? Try these resources:
* **Bug Reports**.
Please file any issues you encounter or fixes you'd like on the
[Calcite Jira](https://issues.apache.org/jira/browse/CALCITE). We welcome
- patches!
+ patches and pull-requests!
* **StackOverflow**.
- [StackOverflow](http://stackoverflow.com) is a wonderful resource for
+ [StackOverflow](http://stackoverflow.com/questions/tagged/calcite) is a wonderful resource for
any developer. Take a look over there to see if someone has answered
your question.
* **Browse the code**.
@@ -122,6 +120,7 @@ As Hadoop Summit, Dublin, 2016
## More talks
+* <a href="https://github.com/julianhyde/share/blob/master/slides/calcite-streaming-sql-xldb-2016.pdf?raw=true">Streaming SQL</a> (XLDB, Menlo Park, 2016)
* <a href="https://github.com/julianhyde/share/blob/master/slides/hive-cbo-seattle-2014.pdf?raw=true">Cost-based optimization in Hive 0.14</a> (Seattle, 2014)
* <a href="https://github.com/julianhyde/share/blob/master/slides/dmmq-summit-2014.pdf?raw=true">Discardable, in-memory materialized query for Hadoop</a> (<a href="https://www.youtube.com/watch?v=CziGOa8GXqI">video</a>) (Hadoop Summit, 2014)
* <a href="https://github.com/julianhyde/share/blob/master/slides/hive-cbo-summit-2014.pdf?raw=true">Cost-based optimization in Hive</a> (<a href="https://www.youtube.com/watch?v=vpG5noIbEFs">video</a>) (Hadoop Summit, 2014)
[5/6] calcite git commit: [CALCITE-1274] Upgrade Spark adapter to
spark-1.6.1
Posted by jh...@apache.org.
[CALCITE-1274] Upgrade Spark adapter to spark-1.6.1
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/d18da01e
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/d18da01e
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/d18da01e
Branch: refs/heads/branch-1.8
Commit: d18da01ed88c772f62123e8420cdefc9b5d76daf
Parents: ec49a0f
Author: Julian Hyde <jh...@apache.org>
Authored: Mon Jun 6 19:46:49 2016 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Jun 8 17:35:03 2016 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../java/org/apache/calcite/adapter/spark/SparkRules.java | 7 ++++---
.../java/org/apache/calcite/adapter/spark/SparkRuntime.java | 4 ++--
3 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/d18da01e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4f8f690..0bcd67c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,7 +112,7 @@ limitations under the License.
<scott-data-hsqldb.version>0.1</scott-data-hsqldb.version>
<servlet.version>3.0.1</servlet.version>
<slf4j.version>1.7.13</slf4j.version>
- <spark.version>0.9.0-incubating</spark.version>
+ <spark.version>1.6.1</spark.version>
<sqlline.version>1.1.9</sqlline.version>
<xalan.version>2.7.1</xalan.version>
<xerces.version>2.9.1</xerces.version>
http://git-wip-us.apache.org/repos/asf/calcite/blob/d18da01e/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
index 2bea47b..3adfcd9 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRules.java
@@ -63,6 +63,7 @@ import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import scala.Tuple2;
@@ -409,9 +410,9 @@ public abstract class SparkRules {
}
//CHECKSTYLE: IGNORE 1
).map(
- new Function<Tuple2<String, List<String>>, Object>() {
- @Override public Object call(Tuple2<String, List<String>> pair) {
- return pair._1() + ":" + pair._2().size();
+ new Function<Tuple2<String, Iterable<String>>, Object>() {
+ @Override public Object call(Tuple2<String, Iterable<String>> pair) {
+ return pair._1() + ":" + Iterables.size(pair._2());
}
}).collect().toString();
System.out.print(s);
http://git-wip-us.apache.org/repos/asf/calcite/blob/d18da01e/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRuntime.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRuntime.java b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRuntime.java
index 0ea1b96..00d9f8d 100644
--- a/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRuntime.java
+++ b/spark/src/main/java/org/apache/calcite/adapter/spark/SparkRuntime.java
@@ -63,8 +63,8 @@ public abstract class SparkRuntime {
/** Combines linq4j {@link org.apache.calcite.linq4j.function.Function}
* and Spark {@link org.apache.spark.api.java.function.FlatMapFunction}. */
public abstract static class CalciteFlatMapFunction<T, R>
- extends FlatMapFunction<T, R>
- implements org.apache.calcite.linq4j.function.Function {
+ implements FlatMapFunction<T, R>,
+ org.apache.calcite.linq4j.function.Function {
}
}
[2/6] calcite git commit: [CALCITE-1277] Rat fails on source
distribution due to git.properties
Posted by jh...@apache.org.
[CALCITE-1277] Rat fails on source distribution due to git.properties
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/a02da271
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/a02da271
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/a02da271
Branch: refs/heads/branch-1.8
Commit: a02da271c2be6f5bf3a226698529748b5ee706ba
Parents: 063f578
Author: Julian Hyde <jh...@apache.org>
Authored: Tue Jun 7 17:05:34 2016 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Jun 8 17:22:29 2016 -0700
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/a02da271/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 02497e8..4f8f690 100644
--- a/pom.xml
+++ b/pom.xml
@@ -474,6 +474,9 @@ limitations under the License.
<!-- Images -->
<exclude>site/img/*.png</exclude>
<exclude>site/favicon.ico</exclude>
+
+ <!-- Not source files, generated during the release process -->
+ <exclude>git.properties</exclude>
</excludes>
</configuration>
</plugin>
[4/6] calcite git commit: [CALCITE-1276] In Druid adapter,
deduce tables and columns if not specified
Posted by jh...@apache.org.
[CALCITE-1276] In Druid adapter, deduce tables and columns if not specified
Deduce tables by calling /druid/coordinator/v1/metadata/datasources;
columns by running a "segmentMetadata" query.
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/435e2030
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/435e2030
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/435e2030
Branch: refs/heads/branch-1.8
Commit: 435e203028a872933c6d8bd6404ebc445552e599
Parents: 23c8e45
Author: Julian Hyde <jh...@apache.org>
Authored: Mon Jun 6 23:03:08 2016 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Jun 8 17:35:03 2016 -0700
----------------------------------------------------------------------
.../org/apache/calcite/model/ModelHandler.java | 19 ++-
.../org/apache/calcite/runtime/HttpUtils.java | 3 +-
druid/pom.xml | 4 +
.../adapter/druid/DruidConnectionImpl.java | 153 ++++++++++++++++++-
.../calcite/adapter/druid/DruidQuery.java | 29 +++-
.../calcite/adapter/druid/DruidSchema.java | 40 ++++-
.../adapter/druid/DruidSchemaFactory.java | 41 ++++-
.../calcite/adapter/druid/DruidTable.java | 65 +++++++-
.../adapter/druid/DruidTableFactory.java | 61 ++++----
.../org/apache/calcite/test/DruidAdapterIT.java | 106 ++++++-------
.../test/resources/druid-foodmart-model.json | 3 +-
druid/src/test/resources/druid-wiki-model.json | 3 +-
.../resources/druid-wiki-no-columns-model.json | 45 ++++++
.../resources/druid-wiki-no-tables-model.json | 33 ++++
site/_docs/druid_adapter.md | 3 +-
15 files changed, 490 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/core/src/main/java/org/apache/calcite/model/ModelHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index 8b451ee..8c7f5fa 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -212,7 +212,7 @@ public class ModelHandler {
jsonSchema.factory);
final Schema schema =
schemaFactory.create(
- parentSchema, jsonSchema.name, operandMap(jsonSchema.operand));
+ parentSchema, jsonSchema.name, operandMap(jsonSchema, jsonSchema.operand));
final SchemaPlus schemaPlus = parentSchema.add(jsonSchema.name, schema);
populateSchema(jsonSchema, schemaPlus);
} catch (Exception e) {
@@ -221,7 +221,8 @@ public class ModelHandler {
}
/** Adds extra entries to an operand to a custom schema. */
- protected Map<String, Object> operandMap(Map<String, Object> operand) {
+ protected Map<String, Object> operandMap(JsonSchema jsonSchema,
+ Map<String, Object> operand) {
if (operand == null) {
return ImmutableMap.of();
}
@@ -238,6 +239,13 @@ public class ModelHandler {
final File file = new File(modelUri);
builder.put(extraOperand.camelName, file.getParentFile());
}
+ break;
+ case TABLES:
+ if (jsonSchema instanceof JsonCustomSchema) {
+ builder.put(extraOperand.camelName,
+ ((JsonCustomSchema) jsonSchema).tables);
+ }
+ break;
}
}
}
@@ -338,7 +346,7 @@ public class ModelHandler {
jsonTable.factory);
final Table table =
tableFactory.create(schema, jsonTable.name,
- operandMap(jsonTable.operand), null);
+ operandMap(null, jsonTable.operand), null);
schema.add(jsonTable.name, table);
} catch (Exception e) {
throw new RuntimeException("Error instantiating " + jsonTable, e);
@@ -428,7 +436,10 @@ public class ModelHandler {
MODEL_URI("modelUri"),
/** Base directory from which to read files. */
- BASE_DIRECTORY("baseDirectory");
+ BASE_DIRECTORY("baseDirectory"),
+
+ /** Tables defined in this schema. */
+ TABLES("tables");
public final String camelName;
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java b/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java
index 011092c..4519281 100644
--- a/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java
+++ b/core/src/main/java/org/apache/calcite/runtime/HttpUtils.java
@@ -126,7 +126,8 @@ public class HttpUtils {
Map<String, String> headers,
int cTimeout,
int rTimeout) throws IOException {
- return executeMethod("POST", url, data, headers, cTimeout, rTimeout);
+ return executeMethod(data == null ? "GET" : "POST", url, data, headers,
+ cTimeout, rTimeout);
}
public static InputStream executeMethod(
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/pom.xml
----------------------------------------------------------------------
diff --git a/druid/pom.xml b/druid/pom.xml
index 22f0c1d..7470774 100644
--- a/druid/pom.xml
+++ b/druid/pom.xml
@@ -57,6 +57,10 @@ limitations under the License.
</dependency>
<dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
index dd462b6..dccda9f 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
@@ -23,14 +23,19 @@ import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.prepare.CalcitePrepareImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Holder;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.CollectionType;
+import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -38,6 +43,7 @@ import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
@@ -49,21 +55,26 @@ import static org.apache.calcite.runtime.HttpUtils.post;
* Implementation of {@link DruidConnection}.
*/
class DruidConnectionImpl implements DruidConnection {
- final String url;
+ private final String url;
+ private final String coordinatorUrl;
- public DruidConnectionImpl(String url) {
- this.url = url;
+ public DruidConnectionImpl(String url, String coordinatorUrl) {
+ this.url = Preconditions.checkNotNull(url);
+ this.coordinatorUrl = Preconditions.checkNotNull(coordinatorUrl);
}
public void request(QueryType queryType, String data, Sink sink,
List<String> fieldNames, Page page) throws IOException {
+ final String url = this.url + "/druid/v2/?pretty";
+ final Map<String, String> requestHeaders =
+ ImmutableMap.of("Content-Type", "application/json");
if (CalcitePrepareImpl.DEBUG) {
System.out.println(data);
}
- final Map<String, String> requestHeaders =
- ImmutableMap.of("Content-Type", "application/json");
- final InputStream in = post(url, data, requestHeaders, 10000, 1800000);
- parse(queryType, in, sink, fieldNames, page);
+ try (InputStream in0 = post(url, data, requestHeaders, 10000, 1800000);
+ InputStream in = traceResponse(in0)) {
+ parse(queryType, in, sink, fieldNames, page);
+ }
}
/** Parses the output of a {@code topN} query, sending the results to a
@@ -290,6 +301,77 @@ class DruidConnectionImpl implements DruidConnection {
};
}
+ /** Reads segment metadata, and populates a list of columns and metrics. */
+ void metadata(String dataSourceName, List<String> intervals,
+ Map<String, SqlTypeName> fieldBuilder, Set<String> metricNameBuilder) {
+ final String url = this.url + "/druid/v2/?pretty";
+ final Map<String, String> requestHeaders =
+ ImmutableMap.of("Content-Type", "application/json");
+ final String data = DruidQuery.metadataQuery(dataSourceName, intervals);
+ if (CalcitePrepareImpl.DEBUG) {
+ System.out.println("Druid: " + data);
+ }
+ try (InputStream in0 = post(url, data, requestHeaders, 10000, 1800000);
+ InputStream in = traceResponse(in0)) {
+ final ObjectMapper mapper = new ObjectMapper();
+ final CollectionType listType =
+ mapper.getTypeFactory().constructCollectionType(List.class,
+ JsonSegmentMetadata.class);
+ final List<JsonSegmentMetadata> list = mapper.readValue(in, listType);
+ in.close();
+ for (JsonSegmentMetadata o : list) {
+ for (Map.Entry<String, JsonColumn> entry : o.columns.entrySet()) {
+ fieldBuilder.put(entry.getKey(), entry.getValue().sqlType());
+ }
+ if (o.aggregators != null) {
+ for (Map.Entry<String, JsonAggregator> entry
+ : o.aggregators.entrySet()) {
+ fieldBuilder.put(entry.getKey(), entry.getValue().sqlType());
+ metricNameBuilder.add(entry.getKey());
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /** Reads data source names from Druid. */
+ Set<String> tableNames() {
+ final Map<String, String> requestHeaders =
+ ImmutableMap.of("Content-Type", "application/json");
+ final String data = null;
+ final String url = coordinatorUrl + "/druid/coordinator/v1/metadata/datasources";
+ if (CalcitePrepareImpl.DEBUG) {
+ System.out.println("Druid: table names" + data + "; " + url);
+ }
+ try (InputStream in0 = post(url, data, requestHeaders, 10000, 1800000);
+ InputStream in = traceResponse(in0)) {
+ final ObjectMapper mapper = new ObjectMapper();
+ final CollectionType listType =
+ mapper.getTypeFactory().constructCollectionType(List.class,
+ String.class);
+ final List<String> list = mapper.readValue(in, listType);
+ return ImmutableSet.copyOf(list);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private InputStream traceResponse(InputStream in) {
+ if (CalcitePrepareImpl.DEBUG) {
+ try {
+ final byte[] bytes = AvaticaUtils.readFullyToBytes(in);
+ in.close();
+ System.out.println("Response: " + new String(bytes));
+ in = new ByteArrayInputStream(bytes);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ return in;
+ }
+
/** A {@link Sink} that is also {@link Runnable}. */
private interface RunnableQueueSink extends Sink, Runnable {
}
@@ -343,6 +425,63 @@ class DruidConnectionImpl implements DruidConnection {
return "{" + pagingIdentifier + ": " + offset + "}";
}
}
+
+
+ /** Result of a "segmentMetadata" call, populated by Jackson. */
+ @SuppressWarnings({ "WeakerAccess", "unused" })
+ private static class JsonSegmentMetadata {
+ public String id;
+ public List<String> intervals;
+ public Map<String, JsonColumn> columns;
+ public int size;
+ public int numRows;
+ public Map<String, JsonAggregator> aggregators;
+ }
+
+ /** Element of the "columns" collection in the result of a
+ * "segmentMetadata" call, populated by Jackson. */
+ @SuppressWarnings({ "WeakerAccess", "unused" })
+ private static class JsonColumn {
+ public String type;
+ public boolean hasMultipleValues;
+ public int size;
+ public Integer cardinality;
+ public String errorMessage;
+
+ SqlTypeName sqlType() {
+ return sqlType(type);
+ }
+
+ static SqlTypeName sqlType(String type) {
+ switch (type) {
+ case "LONG":
+ return SqlTypeName.BIGINT;
+ case "DOUBLE":
+ return SqlTypeName.DOUBLE;
+ case "FLOAT":
+ return SqlTypeName.REAL;
+ case "STRING":
+ return SqlTypeName.VARCHAR;
+ case "hyperUnique":
+ return SqlTypeName.VARBINARY;
+ default:
+ throw new AssertionError("unknown type " + type);
+ }
+ }
+ }
+
+ /** Element of the "aggregators" collection in the result of a
+ * "segmentMetadata" call, populated by Jackson. */
+ @SuppressWarnings({ "WeakerAccess", "unused" })
+ private static class JsonAggregator {
+ public String type;
+ public String name;
+ public String fieldName;
+
+ SqlTypeName sqlType() {
+ return JsonColumn.sqlType(type);
+ }
+ }
}
// End DruidConnectionImpl.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
index 0a9dbec..9a858f9 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
@@ -376,8 +376,7 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
writeFieldIf(generator, "filter", jsonFilter);
writeField(generator, "aggregations", aggregations);
writeFieldIf(generator, "postAggregations", null);
- writeField(generator, "intervals",
- ImmutableList.of(druidTable.interval));
+ writeField(generator, "intervals", druidTable.intervals);
writeFieldIf(generator, "having", null);
generator.writeEndObject();
@@ -389,8 +388,7 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
generator.writeStringField("queryType", "select");
generator.writeStringField("dataSource", druidTable.dataSource);
generator.writeStringField("descending", "false");
- writeField(generator, "intervals",
- ImmutableList.of(druidTable.interval));
+ writeField(generator, "intervals", druidTable.intervals);
writeFieldIf(generator, "filter", jsonFilter);
writeField(generator, "dimensions", translator.dimensions);
writeField(generator, "metrics", translator.metrics);
@@ -445,7 +443,7 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
}
}
- private static void writeField(JsonGenerator generator, String fieldName,
+ static void writeField(JsonGenerator generator, String fieldName,
Object o) throws IOException {
generator.writeFieldName(fieldName);
writeObject(generator, o);
@@ -521,6 +519,24 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
return Pair.of(aboveNodes, belowNodes);
}
+ /** Generates a JSON string to query metadata about a data source. */
+ static String metadataQuery(String dataSourceName, List<String> intervals) {
+ final StringWriter sw = new StringWriter();
+ final JsonFactory factory = new JsonFactory();
+ try {
+ final JsonGenerator generator = factory.createGenerator(sw);
+ generator.writeStartObject();
+ generator.writeStringField("queryType", "segmentMetadata");
+ generator.writeStringField("dataSource", dataSourceName);
+ writeFieldIf(generator, "intervals", intervals);
+ generator.writeEndObject();
+ generator.close();
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ return sw.toString();
+ }
+
/** Druid query specification. */
public static class QuerySpec {
final QueryType queryType;
@@ -660,7 +676,8 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
public void run() throws InterruptedException {
try {
final DruidConnectionImpl connection =
- new DruidConnectionImpl(query.druidTable.schema.url);
+ new DruidConnectionImpl(query.druidTable.schema.url,
+ query.druidTable.schema.coordinatorUrl);
final DruidConnectionImpl.Page page = new DruidConnectionImpl.Page();
int previousOffset;
do {
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java
index bd3539d..28e7086 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchema.java
@@ -18,30 +18,62 @@ package org.apache.calcite.adapter.druid;
import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.impl.AbstractSchema;
+import org.apache.calcite.sql.type.SqlTypeName;
import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nonnull;
/**
* Schema mapped onto a Druid instance.
*/
public class DruidSchema extends AbstractSchema {
final String url;
+ final String coordinatorUrl;
+ private final boolean discoverTables;
/**
* Creates a Druid schema.
*
- * @param url URL of query REST service
+ * @param url URL of query REST service, e.g. "http://localhost:8082"
+ * @param coordinatorUrl URL of coordinator REST service,
+ * e.g. "http://localhost:8081"
+ * @param discoverTables If true, ask Druid what tables exist;
+ * if false, only create tables explicitly in the model
*/
- public DruidSchema(String url) {
+ public DruidSchema(String url, String coordinatorUrl,
+ boolean discoverTables) {
this.url = Preconditions.checkNotNull(url);
+ this.coordinatorUrl = Preconditions.checkNotNull(coordinatorUrl);
+ this.discoverTables = discoverTables;
}
@Override protected Map<String, Table> getTableMap() {
- final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();
- return builder.build();
+ if (!discoverTables) {
+ return ImmutableMap.of();
+ }
+ final DruidConnectionImpl connection =
+ new DruidConnectionImpl(url, coordinatorUrl);
+ return Maps.asMap(ImmutableSet.copyOf(connection.tableNames()),
+ CacheBuilder.<String, Table>newBuilder()
+ .build(new CacheLoader<String, Table>() {
+ public Table load(@Nonnull String tableName) throws Exception {
+ final Map<String, SqlTypeName> fieldMap = new LinkedHashMap<>();
+ final Set<String> metricNameSet = new LinkedHashSet<>();
+ connection.metadata(tableName, null, fieldMap, metricNameSet);
+ return DruidTable.create(DruidSchema.this, tableName, null,
+ fieldMap, metricNameSet, null, connection);
+ }
+ }));
}
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchemaFactory.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchemaFactory.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchemaFactory.java
index f00b932..d35d0bf 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchemaFactory.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidSchemaFactory.java
@@ -20,17 +20,52 @@ import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaFactory;
import org.apache.calcite.schema.SchemaPlus;
+import java.util.List;
import java.util.Map;
/**
* Schema factory that creates Druid schemas.
+ *
+ * <table>
+ * <caption>Druid schema operands</caption>
+ * <tr>
+ * <th>Operand</th>
+ * <th>Description</th>
+ * <th>Required</th>
+ * </tr>
+ * <tr>
+ * <td>url</td>
+ * <td>URL of Druid's query node.
+ * The default is "http://localhost:8082".</td>
+ * <td>No</td>
+ * </tr>
+ * <tr>
+ * <td>coordinatorUrl</td>
+ * <td>URL of Druid's coordinator node.
+ * The default is <code>url</code>, replacing "8082" with "8081",
+ * for example "http://localhost:8081".</td>
+ * <td>No</td>
+ * </tr>
+ * </table>
*/
public class DruidSchemaFactory implements SchemaFactory {
+ /** Default Druid URL. */
+ public static final String DEFAULT_URL = "http://localhost:8082";
+
public Schema create(SchemaPlus parentSchema, String name,
Map<String, Object> operand) {
- Map map = (Map) operand;
- String url = (String) map.get("url");
- return new DruidSchema(url);
+ final Map map = (Map) operand;
+ final String url = map.get("url") instanceof String
+ ? (String) map.get("url")
+ : DEFAULT_URL;
+ final String coordinatorUrl = map.get("coordinatorUrl") instanceof String
+ ? (String) map.get("coordinatorUrl")
+ : url.replace(":8082", ":8081");
+ // "tables" is a hidden attribute, copied in from the enclosing custom
+ // schema
+ final boolean containsTables = map.get("tables") instanceof List
+ && ((List) map.get("tables")).size() > 0;
+ return new DruidSchema(url, coordinatorUrl, !containsTables);
}
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
index 78dcc0d..48258c1 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTable.java
@@ -25,25 +25,33 @@ import org.apache.calcite.rel.logical.LogicalTableScan;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.TranslatableTable;
import org.apache.calcite.schema.impl.AbstractTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.Util;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
* Table mapped onto a Druid table.
*/
public class DruidTable extends AbstractTable implements TranslatableTable {
+ protected static final String DEFAULT_INTERVAL =
+ "1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z";
final DruidSchema schema;
final String dataSource;
final RelProtoDataType protoRowType;
final ImmutableSet<String> metricFieldNames;
- final String interval;
+ final List<String> intervals;
final String timestampFieldName;
/**
@@ -53,18 +61,50 @@ public class DruidTable extends AbstractTable implements TranslatableTable {
* @param dataSource Druid data source name
* @param protoRowType Field names and types
* @param metricFieldNames Names of fields that are metrics
- * @param interval Default interval if query does not constrain the time
+ * @param intervals Default interval if query does not constrain the time, or null
* @param timestampFieldName Name of the column that contains the time
*/
public DruidTable(DruidSchema schema, String dataSource,
- RelProtoDataType protoRowType, Set<String> metricFieldNames, String interval,
+ RelProtoDataType protoRowType, Set<String> metricFieldNames, List<String> intervals,
String timestampFieldName) {
this.timestampFieldName = Preconditions.checkNotNull(timestampFieldName);
this.schema = Preconditions.checkNotNull(schema);
this.dataSource = Preconditions.checkNotNull(dataSource);
this.protoRowType = protoRowType;
this.metricFieldNames = ImmutableSet.copyOf(metricFieldNames);
- this.interval = Preconditions.checkNotNull(interval);
+ this.intervals = Preconditions.checkNotNull(
+ Util.first(intervals, ImmutableList.of(DEFAULT_INTERVAL)));
+ }
+
+ /** Creates a {@link DruidTable}
+ *
+ * @param druidSchema Druid schema
+ * @param dataSourceName Data source name in Druid, also table name
+ * @param intervals Intervals, or null to use default
+ * @param fieldMap Mutable map of fields (dimensions plus metrics);
+ * may be partially populated already
+ * @param metricNameSet Mutable set of metric names;
+ * may be partially populated already
+ * @param timestampColumnName Name of timestamp column, or null
+ * @param connection If not null, use this connection to find column
+ * definitions
+ * @return A table
+ */
+ static Table create(DruidSchema druidSchema, String dataSourceName,
+ List<String> intervals, Map<String, SqlTypeName> fieldMap,
+ Set<String> metricNameSet, String timestampColumnName,
+ DruidConnectionImpl connection) {
+ if (connection != null) {
+ connection.metadata(dataSourceName, intervals, fieldMap, metricNameSet);
+ }
+ final ImmutableMap<String, SqlTypeName> fields =
+ ImmutableMap.copyOf(fieldMap);
+ if (timestampColumnName == null) {
+ timestampColumnName = Iterables.get(fieldMap.keySet(), 0);
+ }
+ return new DruidTable(druidSchema, dataSourceName,
+ new MapRelProtoDataType(fields), ImmutableSet.copyOf(metricNameSet),
+ intervals, Util.first(timestampColumnName, "__time"));
}
public RelDataType getRowType(RelDataTypeFactory typeFactory) {
@@ -84,6 +124,23 @@ public class DruidTable extends AbstractTable implements TranslatableTable {
ImmutableList.<RelNode>of(scan));
}
+ /** Creates a {@link RelDataType} from a map of
+ * field names and types. */
+ private static class MapRelProtoDataType implements RelProtoDataType {
+ private final ImmutableMap<String, SqlTypeName> fields;
+
+ MapRelProtoDataType(ImmutableMap<String, SqlTypeName> fields) {
+ this.fields = fields;
+ }
+
+ public RelDataType apply(RelDataTypeFactory typeFactory) {
+ final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
+ for (Map.Entry<String, SqlTypeName> field : fields.entrySet()) {
+ builder.add(field.getKey(), field.getValue()).nullable(true);
+ }
+ return builder.build();
+ }
+ }
}
// End DruidTable.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
index 756e558..2453fcb 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidTableFactory.java
@@ -17,17 +17,15 @@
package org.apache.calcite.adapter.druid;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rel.type.RelProtoDataType;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.TableFactory;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Util;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableList;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -39,6 +37,11 @@ import java.util.Set;
* <p>A table corresponds to what Druid calls a "data source".
*/
public class DruidTableFactory implements TableFactory {
+ @SuppressWarnings("unused")
+ public static final DruidTableFactory INSTANCE = new DruidTableFactory();
+
+ private DruidTableFactory() {}
+
public Table create(SchemaPlus schema, String name, Map operand,
RelDataType rowType) {
final DruidSchema druidSchema = schema.unwrap(DruidSchema.class);
@@ -46,17 +49,18 @@ public class DruidTableFactory implements TableFactory {
final String dataSource = (String) operand.get("dataSource");
final Set<String> metricNameBuilder = new LinkedHashSet<>();
String timestampColumnName = (String) operand.get("timestampColumn");
- final ImmutableMap.Builder<String, SqlTypeName> fieldBuilder =
- ImmutableMap.builder();
- if (operand.get("dimensions") != null) {
+ final Map<String, SqlTypeName> fieldBuilder = new LinkedHashMap<>();
+ final Object dimensionsRaw = operand.get("dimensions");
+ if (dimensionsRaw instanceof List) {
//noinspection unchecked
- final List<String> dimensions = (List<String>) operand.get("dimensions");
+ final List<String> dimensions = (List<String>) dimensionsRaw;
for (String dimension : dimensions) {
fieldBuilder.put(dimension, SqlTypeName.VARCHAR);
}
}
- if (operand.get("metrics") != null) {
- final List metrics = (List) operand.get("metrics");
+ final Object metricsRaw = operand.get("metrics");
+ if (metricsRaw instanceof List) {
+ final List metrics = (List) metricsRaw;
for (Object metric : metrics) {
final SqlTypeName sqlTypeName;
final String metricName;
@@ -83,32 +87,23 @@ public class DruidTableFactory implements TableFactory {
metricNameBuilder.add(metricName);
}
}
- fieldBuilder.put(timestampColumnName, SqlTypeName.VARCHAR);
- final ImmutableMap<String, SqlTypeName> fields = fieldBuilder.build();
- String interval = Util.first((String) operand.get("interval"),
- "1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z");
- return new DruidTable(druidSchema, Util.first(dataSource, name),
- new MapRelProtoDataType(fields),
- ImmutableSet.copyOf(metricNameBuilder), interval, timestampColumnName);
- }
-
- /** Creates a {@link org.apache.calcite.rel.type.RelDataType} from a map of
- * field names and types. */
- private static class MapRelProtoDataType implements RelProtoDataType {
- private final ImmutableMap<String, SqlTypeName> fields;
-
- public MapRelProtoDataType(ImmutableMap<String, SqlTypeName> fields) {
- this.fields = fields;
+ if (timestampColumnName != null) {
+ fieldBuilder.put(timestampColumnName, SqlTypeName.VARCHAR);
}
-
- public RelDataType apply(RelDataTypeFactory typeFactory) {
- final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
- for (Map.Entry<String, SqlTypeName> field : fields.entrySet()) {
- builder.add(field.getKey(), field.getValue()).nullable(true);
- }
- return builder.build();
+ final String dataSourceName = Util.first(dataSource, name);
+ DruidConnectionImpl c;
+ if (dimensionsRaw == null || metricsRaw == null) {
+ c = new DruidConnectionImpl(druidSchema.url, druidSchema.url.replace(":8082", ":8081"));
+ } else {
+ c = null;
}
+ final Object interval = operand.get("interval");
+ final List<String> intervals = interval instanceof String
+ ? ImmutableList.of((String) interval) : null;
+ return DruidTable.create(druidSchema, dataSourceName, intervals,
+ fieldBuilder, metricNameBuilder, timestampColumnName, c);
}
+
}
// End DruidTableFactory.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index 34717b6..d59ef24 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -22,18 +22,16 @@ import org.apache.calcite.util.Util;
import com.google.common.base.Function;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
import org.junit.Ignore;
import org.junit.Test;
+import java.net.URL;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Collections;
import java.util.List;
import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -60,18 +58,24 @@ import static org.junit.Assert.assertTrue;
* </ul>
*/
public class DruidAdapterIT {
- /** Connection factory based on the "druid-foodmart" model. */
- public static final ImmutableMap<String, String> FOODMART =
- ImmutableMap.of("model",
- DruidAdapterIT.class.getResource("/druid-foodmart-model.json")
- .getPath());
+ /** URL of the "druid-foodmart" model. */
+ public static final URL FOODMART =
+ DruidAdapterIT.class.getResource("/druid-foodmart-model.json");
- /** Connection factory based on the "druid-wiki" model
+ /** URL of the "druid-wiki" model
* and the "wikiticker" data set. */
- public static final ImmutableMap<String, String> WIKI =
- ImmutableMap.of("model",
- DruidAdapterIT.class.getResource("/druid-wiki-model.json")
- .getPath());
+ public static final URL WIKI =
+ DruidAdapterIT.class.getResource("/druid-wiki-model.json");
+
+ /** URL of the "druid-wiki-no-columns" model
+ * and the "wikiticker" data set. */
+ public static final URL WIKI_AUTO =
+ DruidAdapterIT.class.getResource("/druid-wiki-no-columns-model.json");
+
+ /** URL of the "druid-wiki-no-tables" model
+ * and the "wikiticker" data set. */
+ public static final URL WIKI_AUTO2 =
+ DruidAdapterIT.class.getResource("/druid-wiki-no-tables-model.json");
/** Whether to run Druid tests. Enabled by default, however test is only
* included if "it" profile is activated ({@code -Pit}). To disable,
@@ -100,48 +104,17 @@ public class DruidAdapterIT {
};
}
- /** Similar to {@link CalciteAssert#checkResultUnordered}, but filters strings
- * before comparing them. */
- static Function<ResultSet, Void> checkResultUnordered(
- final String... lines) {
- return new Function<ResultSet, Void>() {
- public Void apply(ResultSet resultSet) {
- try {
- final List<String> expectedList = Lists.newArrayList(lines);
- Collections.sort(expectedList);
-
- final List<String> actualList = Lists.newArrayList();
- CalciteAssert.toStringList(resultSet, actualList);
- for (int i = 0; i < actualList.size(); i++) {
- String s = actualList.get(i);
- actualList.set(i,
- s.replaceAll("\\.0;", ";").replaceAll("\\.0$", ""));
- }
- Collections.sort(actualList);
-
- assertThat(actualList, equalTo(expectedList));
- return null;
- } catch (SQLException e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
-
- /** Creates a query against the {@link #FOODMART} data set. */
- private CalciteAssert.AssertQuery sql(String sql) {
+ /** Creates a query against a data set given by a map. */
+ private CalciteAssert.AssertQuery sql(String sql, URL url) {
return CalciteAssert.that()
.enable(enabled())
- .with(FOODMART)
+ .with(ImmutableMap.of("model", url.getPath()))
.query(sql);
}
- /** Creates a query against the {@link #WIKI} data set. */
- private CalciteAssert.AssertQuery wiki(String sql) {
- return CalciteAssert.that()
- .enable(enabled())
- .with(WIKI)
- .query(sql);
+ /** Creates a query against the {@link #FOODMART} data set. */
+ private CalciteAssert.AssertQuery sql(String sql) {
+ return sql(sql, FOODMART);
}
/** Tests a query against the {@link #WIKI} data set.
@@ -152,8 +125,36 @@ public class DruidAdapterIT {
final String explain = "PLAN="
+ "EnumerableInterpreter\n"
+ " DruidQuery(table=[[wiki, wiki]], filter=[=(CAST($12):VARCHAR(13) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'Jeremy Corbyn')], groups=[{4}], aggs=[[]])\n";
+ checkSelectDistinctWiki(WIKI, "wiki")
+ .explainContains(explain);
+ }
+
+ @Test public void testSelectDistinctWikiNoColumns() {
+ final String explain = "PLAN="
+ + "EnumerableInterpreter\n"
+ + " DruidQuery(table=[[wiki, wiki]], filter=[=(CAST($18):VARCHAR(13) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'Jeremy Corbyn')], groups=[{8}], aggs=[[]])\n";
+ checkSelectDistinctWiki(WIKI_AUTO, "wiki")
+ .explainContains(explain);
+ }
+
+ @Test public void testSelectDistinctWikiNoTables() {
+ // Compared to testSelectDistinctWiki, table name is different (because it
+ // is the raw dataSource name from Druid) and the field offsets are
+ // different. This is expected.
+ final String explain = "PLAN="
+ + "EnumerableInterpreter\n"
+ + " DruidQuery(table=[[wiki, wikiticker]], filter=[=(CAST($17):VARCHAR(13) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'Jeremy Corbyn')], groups=[{7}], aggs=[[]])\n";
+ checkSelectDistinctWiki(WIKI_AUTO2, "wikiticker")
+ .explainContains(explain);
+
+ // Because no tables are declared, foodmart is automatically present.
+ sql("select count(*) as c from \"foodmart\"", WIKI_AUTO2)
+ .returnsUnordered("C=86829");
+ }
+
+ private CalciteAssert.AssertQuery checkSelectDistinctWiki(URL url, String tableName) {
final String sql = "select distinct \"countryName\"\n"
- + "from \"wiki\"\n"
+ + "from \"" + tableName + "\"\n"
+ "where \"page\" = 'Jeremy Corbyn'";
final String druidQuery = "{'queryType':'groupBy',"
+ "'dataSource':'wikiticker','granularity':'all',"
@@ -161,10 +162,9 @@ public class DruidAdapterIT {
+ "'filter':{'type':'selector','dimension':'page','value':'Jeremy Corbyn'},"
+ "'aggregations':[{'type':'longSum','name':'unit_sales','fieldName':'unit_sales'}],"
+ "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}";
- wiki(sql)
+ return sql(sql, url)
.returnsUnordered("countryName=United Kingdom",
"countryName=null")
- .explainContains(explain)
.queryContains(druidChecker(druidQuery));
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/test/resources/druid-foodmart-model.json
----------------------------------------------------------------------
diff --git a/druid/src/test/resources/druid-foodmart-model.json b/druid/src/test/resources/druid-foodmart-model.json
index 68102f0..8e0e14e 100644
--- a/druid/src/test/resources/druid-foodmart-model.json
+++ b/druid/src/test/resources/druid-foodmart-model.json
@@ -23,7 +23,8 @@
"name": "foodmart",
"factory": "org.apache.calcite.adapter.druid.DruidSchemaFactory",
"operand": {
- "url": "http://localhost:8082/druid/v2/?pretty"
+ "url": "http://localhost:8082",
+ "coordinatorUrl": "http://localhost:8081"
},
"tables": [
{
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/test/resources/druid-wiki-model.json
----------------------------------------------------------------------
diff --git a/druid/src/test/resources/druid-wiki-model.json b/druid/src/test/resources/druid-wiki-model.json
index eb13494..f345788 100644
--- a/druid/src/test/resources/druid-wiki-model.json
+++ b/druid/src/test/resources/druid-wiki-model.json
@@ -23,7 +23,8 @@
"name": "wiki",
"factory": "org.apache.calcite.adapter.druid.DruidSchemaFactory",
"operand": {
- "url": "http://localhost:8082/druid/v2/?pretty"
+ "url": "http://localhost:8082",
+ "coordinatorUrl": "http://localhost:8081"
},
"tables": [
{
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/test/resources/druid-wiki-no-columns-model.json
----------------------------------------------------------------------
diff --git a/druid/src/test/resources/druid-wiki-no-columns-model.json b/druid/src/test/resources/druid-wiki-no-columns-model.json
new file mode 100644
index 0000000..c9231c4
--- /dev/null
+++ b/druid/src/test/resources/druid-wiki-no-columns-model.json
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Druid model that has one table defined, and table has no columns, so adapter
+ * needs to discover columns.
+ */
+{
+ "version": "1.0",
+ "defaultSchema": "wiki",
+ "schemas": [
+ {
+ "type": "custom",
+ "name": "wiki",
+ "factory": "org.apache.calcite.adapter.druid.DruidSchemaFactory",
+ "operand": {
+ "url": "http://localhost:8082",
+ "coordinatorUrl": "http://localhost:8081"
+ },
+ "tables": [
+ {
+ "name": "wiki",
+ "factory": "org.apache.calcite.adapter.druid.DruidTableFactory",
+ "operand": {
+ "dataSource": "wikiticker",
+ "interval": "1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z",
+ "timestampColumn": "time"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/druid/src/test/resources/druid-wiki-no-tables-model.json
----------------------------------------------------------------------
diff --git a/druid/src/test/resources/druid-wiki-no-tables-model.json b/druid/src/test/resources/druid-wiki-no-tables-model.json
new file mode 100644
index 0000000..5b22ef9
--- /dev/null
+++ b/druid/src/test/resources/druid-wiki-no-tables-model.json
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Druid model where there are no table definitions.
+ */
+{
+ "version": "1.0",
+ "defaultSchema": "wiki",
+ "schemas": [
+ {
+ "type": "custom",
+ "name": "wiki",
+ "factory": "org.apache.calcite.adapter.druid.DruidSchemaFactory",
+ "operand": {
+ "url": "http://localhost:8082",
+ "coordinatorUrl": "http://localhost:8081"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/calcite/blob/435e2030/site/_docs/druid_adapter.md
----------------------------------------------------------------------
diff --git a/site/_docs/druid_adapter.md b/site/_docs/druid_adapter.md
index 3a330a6..9de2fb9 100644
--- a/site/_docs/druid_adapter.md
+++ b/site/_docs/druid_adapter.md
@@ -51,7 +51,8 @@ A basic example of a model file is given below:
"name": "wiki",
"factory": "org.apache.calcite.adapter.druid.DruidSchemaFactory",
"operand": {
- "url": "http://localhost:8082/druid/v2/?pretty"
+ "url": "http://localhost:8082",
+ "coordinatorUrl": "http://localhost:8081"
},
"tables": [
{
[3/6] calcite git commit: [CALCITE-1281] Druid adapter wrongly
returns all numeric values as int or float
Posted by jh...@apache.org.
[CALCITE-1281] Druid adapter wrongly returns all numeric values as int or float
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/ec49a0fa
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/ec49a0fa
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/ec49a0fa
Branch: refs/heads/branch-1.8
Commit: ec49a0fa37195bb4b34945b53ce39b27d558d6ab
Parents: 435e203
Author: Julian Hyde <jh...@apache.org>
Authored: Wed Jun 8 17:03:17 2016 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Jun 8 17:35:03 2016 -0700
----------------------------------------------------------------------
.../adapter/druid/DruidConnectionImpl.java | 72 +++++++++++++++-----
.../calcite/adapter/druid/DruidQuery.java | 28 +++++++-
.../org/apache/calcite/test/DruidAdapterIT.java | 33 +++++++--
3 files changed, 112 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/ec49a0fa/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
index dccda9f..7520d70 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidConnectionImpl.java
@@ -22,6 +22,7 @@ import org.apache.calcite.interpreter.Sink;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.linq4j.tree.Primitive;
import org.apache.calcite.prepare.CalcitePrepareImpl;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Holder;
@@ -40,6 +41,7 @@ import com.google.common.collect.ImmutableSet;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -58,13 +60,24 @@ class DruidConnectionImpl implements DruidConnection {
private final String url;
private final String coordinatorUrl;
- public DruidConnectionImpl(String url, String coordinatorUrl) {
+ DruidConnectionImpl(String url, String coordinatorUrl) {
this.url = Preconditions.checkNotNull(url);
this.coordinatorUrl = Preconditions.checkNotNull(coordinatorUrl);
}
+ /** Executes a query request.
+ *
+ * @param queryType Query type
+ * @param data Data to post
+ * @param sink Sink to which to send the parsed rows
+ * @param fieldNames Names of fields
+ * @param fieldTypes Types of fields (never null, but elements may be null)
+ * @param page Page definition (in/out)
+ * @throws IOException on error
+ */
public void request(QueryType queryType, String data, Sink sink,
- List<String> fieldNames, Page page) throws IOException {
+ List<String> fieldNames, List<Primitive> fieldTypes, Page page)
+ throws IOException {
final String url = this.url + "/druid/v2/?pretty";
final Map<String, String> requestHeaders =
ImmutableMap.of("Content-Type", "application/json");
@@ -73,14 +86,14 @@ class DruidConnectionImpl implements DruidConnection {
}
try (InputStream in0 = post(url, data, requestHeaders, 10000, 1800000);
InputStream in = traceResponse(in0)) {
- parse(queryType, in, sink, fieldNames, page);
+ parse(queryType, in, sink, fieldNames, fieldTypes, page);
}
}
/** Parses the output of a {@code topN} query, sending the results to a
* {@link Sink}. */
private void parse(QueryType queryType, InputStream in, Sink sink,
- List<String> fieldNames, Page page) {
+ List<String> fieldNames, List<Primitive> fieldTypes, Page page) {
final JsonFactory factory = new JsonFactory();
final Row.RowBuilder rowBuilder = Row.newBuilder(fieldNames.size());
@@ -105,7 +118,7 @@ class DruidConnectionImpl implements DruidConnection {
&& parser.nextToken() == JsonToken.START_ARRAY) {
while (parser.nextToken() == JsonToken.START_OBJECT) {
// loop until token equal to "}"
- parseFields(fieldNames, rowBuilder, parser);
+ parseFields(fieldNames, fieldTypes, rowBuilder, parser);
sink.send(rowBuilder.build());
rowBuilder.reset();
}
@@ -145,7 +158,7 @@ class DruidConnectionImpl implements DruidConnection {
if (parser.nextToken() == JsonToken.FIELD_NAME
&& parser.getCurrentName().equals("event")
&& parser.nextToken() == JsonToken.START_OBJECT) {
- parseFields(fieldNames, rowBuilder, parser);
+ parseFields(fieldNames, fieldTypes, rowBuilder, parser);
sink.send(rowBuilder.build());
rowBuilder.reset();
}
@@ -165,7 +178,7 @@ class DruidConnectionImpl implements DruidConnection {
if (parser.nextToken() == JsonToken.FIELD_NAME
&& parser.getCurrentName().equals("event")
&& parser.nextToken() == JsonToken.START_OBJECT) {
- parseFields(fieldNames, rowBuilder, parser);
+ parseFields(fieldNames, fieldTypes, rowBuilder, parser);
sink.send(rowBuilder.build());
rowBuilder.reset();
}
@@ -178,15 +191,15 @@ class DruidConnectionImpl implements DruidConnection {
}
}
- private void parseFields(List<String> fieldNames, Row.RowBuilder rowBuilder,
- JsonParser parser) throws IOException {
+ private void parseFields(List<String> fieldNames, List<Primitive> fieldTypes,
+ Row.RowBuilder rowBuilder, JsonParser parser) throws IOException {
while (parser.nextToken() == JsonToken.FIELD_NAME) {
- parseField(fieldNames, rowBuilder, parser);
+ parseField(fieldNames, fieldTypes, rowBuilder, parser);
}
}
- private void parseField(List<String> fieldNames, Row.RowBuilder rowBuilder,
- JsonParser parser) throws IOException {
+ private void parseField(List<String> fieldNames, List<Primitive> fieldTypes,
+ Row.RowBuilder rowBuilder, JsonParser parser) throws IOException {
final String fieldName = parser.getCurrentName();
// Move to next token, which is name's value
@@ -197,10 +210,35 @@ class DruidConnectionImpl implements DruidConnection {
}
switch (token) {
case VALUE_NUMBER_INT:
- rowBuilder.set(i, parser.getIntValue());
- break;
case VALUE_NUMBER_FLOAT:
- rowBuilder.set(i, parser.getDoubleValue());
+ Primitive type = fieldTypes.get(i);
+ if (type == null) {
+ if (token == JsonToken.VALUE_NUMBER_INT) {
+ type = Primitive.INT;
+ } else {
+ type = Primitive.FLOAT;
+ }
+ }
+ switch (type) {
+ case BYTE:
+ rowBuilder.set(i, parser.getIntValue());
+ break;
+ case SHORT:
+ rowBuilder.set(i, parser.getShortValue());
+ break;
+ case INT:
+ rowBuilder.set(i, parser.getIntValue());
+ break;
+ case LONG:
+ rowBuilder.set(i, parser.getLongValue());
+ break;
+ case FLOAT:
+ rowBuilder.set(i, parser.getFloatValue());
+ break;
+ case DOUBLE:
+ rowBuilder.set(i, parser.getDoubleValue());
+ break;
+ }
break;
case VALUE_TRUE:
rowBuilder.set(i, true);
@@ -287,7 +325,9 @@ class DruidConnectionImpl implements DruidConnection {
public void run() {
try {
final Page page = new Page();
- request(queryType, request, this, fieldNames, page);
+ final List<Primitive> fieldTypes =
+ Collections.nCopies(fieldNames.size(), null);
+ request(queryType, request, this, fieldNames, fieldTypes, page);
enumerator.done.set(true);
} catch (Throwable e) {
enumerator.throwableHolder.set(e);
http://git-wip-us.apache.org/repos/asf/calcite/blob/ec49a0fa/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
index 9a858f9..a3fdf78 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
@@ -24,6 +24,7 @@ import org.apache.calcite.interpreter.Interpreter;
import org.apache.calcite.interpreter.Node;
import org.apache.calcite.interpreter.Sink;
import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.tree.Primitive;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
@@ -41,6 +42,7 @@ import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
@@ -674,6 +676,10 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
}
public void run() throws InterruptedException {
+ final List<Primitive> fieldTypes = new ArrayList<>();
+ for (RelDataTypeField field : query.getRowType().getFieldList()) {
+ fieldTypes.add(getPrimitive(field));
+ }
try {
final DruidConnectionImpl connection =
new DruidConnectionImpl(query.druidTable.schema.url,
@@ -685,12 +691,32 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
final String queryString =
querySpec.getQueryString(page.pagingIdentifier, page.offset);
connection.request(querySpec.queryType, queryString, sink,
- querySpec.fieldNames, page);
+ querySpec.fieldNames, fieldTypes, page);
} while (page.pagingIdentifier != null && page.offset > previousOffset);
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
+
+ private Primitive getPrimitive(RelDataTypeField field) {
+ switch (field.getType().getSqlTypeName()) {
+ case BIGINT:
+ return Primitive.LONG;
+ case INTEGER:
+ return Primitive.INT;
+ case SMALLINT:
+ return Primitive.SHORT;
+ case TINYINT:
+ return Primitive.BYTE;
+ case REAL:
+ return Primitive.FLOAT;
+ case DOUBLE:
+ case FLOAT:
+ return Primitive.DOUBLE;
+ default:
+ return null;
+ }
+ }
}
/** Object that knows how to write itself to a
http://git-wip-us.apache.org/repos/asf/calcite/blob/ec49a0fa/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index d59ef24..fee3e3d 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -200,6 +200,29 @@ public class DruidAdapterIT {
.explainContains(explain);
}
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-1281">[CALCITE-1281]
+ * Druid adapter wrongly returns all numeric values as int or float</a>. */
+ @Test public void testSelectCount() {
+ final String sql = "select count(*) as c from \"foodmart\"";
+ sql(sql)
+ .returns(new Function<ResultSet, Void>() {
+ public Void apply(ResultSet input) {
+ try {
+ assertThat(input.next(), is(true));
+ assertThat(input.getInt(1), is(86829));
+ assertThat(input.getLong(1), is(86829L));
+ assertThat(input.getString(1), is("86829"));
+ assertThat(input.wasNull(), is(false));
+ assertThat(input.next(), is(false));
+ return null;
+ } catch (SQLException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ });
+ }
+
@Test public void testSort() {
// Note: We do not push down SORT yet
final String explain = "PLAN="
@@ -455,7 +478,6 @@ public class DruidAdapterIT {
"C=21610; state_province=OR");
}
- @Ignore("TODO: fix invalid cast from Integer to Long")
@Test public void testGroupByAvgSumCount() {
final String sql = "select \"state_province\",\n"
+ " avg(\"unit_sales\") as a,\n"
@@ -467,11 +489,14 @@ public class DruidAdapterIT {
+ "order by 1";
String druidQuery = "'aggregations':["
+ "{'type':'longSum','name':'$f1','fieldName':'unit_sales'},"
- + "{'type':'count','name':'$f2'}]";
+ + "{'type':'count','name':'$f2','fieldName':'unit_sales'},"
+ + "{'type':'count','name':'C','fieldName':'store_sqft'},"
+ + "{'type':'count','name':'C0'}],"
+ + "'intervals':['1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z']}";
sql(sql)
.limit(2)
- .returnsUnordered("state_province=CA; A=3; S=74748; C=23190; C0=23190",
- "state_province=OR; A=3; S=67659; C=19027; C0=19027")
+ .returnsUnordered("state_province=CA; A=3; S=74748; C=24441; C0=24441",
+ "state_province=OR; A=3; S=67659; C=21610; C0=21610")
.queryContains(druidChecker(druidQuery));
}