You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2017/03/17 21:32:38 UTC
[1/2] calcite git commit: [CALCITE-1706] Disable
DruidAggregateFilterTransposeRule,
because it causes fine-grained aggregations to be pushed to Druid
Repository: calcite
Updated Branches:
refs/heads/master 7f46cd5f0 -> 0e11746ac
[CALCITE-1706] Disable DruidAggregateFilterTransposeRule, because it causes fine-grained aggregations to be pushed to Druid
Also remove some more CAST calls erroneously introduced by [CALCITE-1684].
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/6b54b6ec
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/6b54b6ec
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/6b54b6ec
Branch: refs/heads/master
Commit: 6b54b6ec1718040980ed9ea60faa0e1814ae3ea4
Parents: 7f46cd5
Author: Julian Hyde <jh...@apache.org>
Authored: Fri Mar 17 11:47:00 2017 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Fri Mar 17 11:47:00 2017 -0700
----------------------------------------------------------------------
.../calcite/adapter/druid/DruidRules.java | 16 ++++++++---
.../org/apache/calcite/test/DruidAdapterIT.java | 29 ++++++++++++--------
2 files changed, 29 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/6b54b6ec/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
index bb15921..b22809c 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
@@ -92,10 +92,18 @@ public class DruidRules {
public static final DruidFilterAggregateTransposeRule FILTER_AGGREGATE_TRANSPOSE_RULE =
new DruidFilterAggregateTransposeRule();
- public static final List<RelOptRule> RULES = ImmutableList.of(FILTER,
- PROJECT_FILTER_TRANSPOSE_RULE, AGGREGATE_FILTER_TRANSPOSE_RULE, PROJECT_AGGREGATE,
- PROJECT, AGGREGATE, FILTER_AGGREGATE_TRANSPOSE_RULE, FILTER_PROJECT_TRANSPOSE_RULE,
- PROJECT_SORT, SORT, SORT_PROJECT);
+ public static final List<RelOptRule> RULES =
+ ImmutableList.of(FILTER,
+ PROJECT_FILTER_TRANSPOSE_RULE,
+ // Disabled, per
+ // [CALCITE-1706] DruidAggregateFilterTransposeRule
+ // causes very fine-grained aggregations to be pushed to Druid
+ // AGGREGATE_FILTER_TRANSPOSE_RULE,
+ PROJECT_AGGREGATE,
+ PROJECT, AGGREGATE,
+ FILTER_AGGREGATE_TRANSPOSE_RULE,
+ FILTER_PROJECT_TRANSPOSE_RULE,
+ PROJECT_SORT, SORT, SORT_PROJECT);
/** Predicate that returns whether Druid can not handle an aggregate. */
private static final Predicate<Aggregate> BAD_AGG =
http://git-wip-us.apache.org/repos/asf/calcite/blob/6b54b6ec/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index bcce57e..4ee32a5 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -89,7 +89,7 @@ public class DruidAdapterIT {
Util.getBooleanProperty("calcite.test.druid", true);
private static final String VARCHAR_TYPE =
- "VARCHAR(1) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"";
+ "VARCHAR CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"";
/** Whether to run this test. */
protected boolean enabled() {
@@ -132,7 +132,9 @@ public class DruidAdapterIT {
@Test public void testSelectDistinctWiki() {
final String explain = "PLAN="
+ "EnumerableInterpreter\n"
- + " DruidQuery(table=[[wiki, wiki]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], filter=[=(CAST($13):VARCHAR(13) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'Jeremy Corbyn')], groups=[{5}], aggs=[[]])\n";
+ + " DruidQuery(table=[[wiki, wiki]], "
+ + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], "
+ + "filter=[=($13, 'Jeremy Corbyn')], groups=[{5}], aggs=[[]])\n";
checkSelectDistinctWiki(WIKI, "wiki")
.explainContains(explain);
}
@@ -140,7 +142,9 @@ public class DruidAdapterIT {
@Test public void testSelectDistinctWikiNoColumns() {
final String explain = "PLAN="
+ "EnumerableInterpreter\n"
- + " DruidQuery(table=[[wiki, wiki]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], filter=[=(CAST($17):VARCHAR(13) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'Jeremy Corbyn')], groups=[{7}], aggs=[[]])\n";
+ + " DruidQuery(table=[[wiki, wiki]], "
+ + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], "
+ + "filter=[=($17, 'Jeremy Corbyn')], groups=[{7}], aggs=[[]])\n";
checkSelectDistinctWiki(WIKI_AUTO, "wiki")
.explainContains(explain);
}
@@ -154,8 +158,10 @@ public class DruidAdapterIT {
+ "from \"wikiticker\"\n"
+ "where \"page\" = 'Jeremy Corbyn'";
final String explain = "PLAN="
- + "EnumerableInterpreter\n"
- + " DruidQuery(table=[[wiki, wikiticker]], intervals=[[1900-01-01T00:00:00.000/3000-01-01T00:00:00.000]], filter=[=(CAST($17):VARCHAR(13) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'Jeremy Corbyn')], groups=[{7}], aggs=[[]])\n";
+ + "EnumerableInterpreter\n"
+ + " DruidQuery(table=[[wiki, wikiticker]], "
+ + "intervals=[[1900-01-01T00:00:00.000/3000-01-01T00:00:00.000]], "
+ + "filter=[=($17, 'Jeremy Corbyn')], groups=[{7}], aggs=[[]])\n";
final String druidQuery = "{'queryType':'groupBy',"
+ "'dataSource':'wikiticker','granularity':'all',"
+ "'dimensions':['countryName'],'limitSpec':{'type':'default'},"
@@ -1302,12 +1308,11 @@ public class DruidAdapterIT {
+ "'granularity':'all',"
+ "'pagingSpec':{'threshold':16384,'fromNext':true},'context':{'druid.query.fetch':false}}";
final String explain = "PLAN=EnumerableInterpreter\n"
- + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]],"
- + " filter=[AND(=(CAST($3):VARCHAR(24) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'High Top Dried Mushrooms'),"
- + " OR(=($87, 'Q2'),"
- + " =($87, 'Q3')),"
- + " =(CAST($30):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'WA'))],"
- + " projects=[[$30, $29, $3]])\n";
+ + " DruidQuery(table=[[foodmart, foodmart]], "
+ + "intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], "
+ + "filter=[AND(=($3, 'High Top Dried Mushrooms'), "
+ + "OR(=($87, 'Q2'), =($87, 'Q3')), =($30, 'WA'))], "
+ + "projects=[[$30, $29, $3]])\n";
sql(sql)
.queryContains(druidChecker(druidQuery))
.explainContains(explain)
@@ -1354,7 +1359,7 @@ public class DruidAdapterIT {
+ "from \"foodmart\"\n"
+ "where 'High Top Dried Mushrooms' = \"product_name\"";
final String explain = "EnumerableInterpreter\n"
- + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], filter=[=('High Top Dried Mushrooms', CAST($3):VARCHAR(24) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\")], projects=[[$30]])";
+ + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], filter=[=('High Top Dried Mushrooms', $3)], projects=[[$30]])";
final String druidQuery = "'filter':{'type':'selector','dimension':'product_name',"
+ "'value':'High Top Dried Mushrooms'}";
sql(sql)
[2/2] calcite git commit: [CALCITE-1704] Execute queries on CSV files
using simple sqlline command
Posted by jh...@apache.org.
[CALCITE-1704] Execute queries on CSV files using simple sqlline command
Document how to use file adapter without an explicit model;
make sure that baseDirectory is set even without explicit model.
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/0e11746a
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/0e11746a
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/0e11746a
Branch: refs/heads/master
Commit: 0e11746aca3e0abf4e60fd1545eb89ecf174ca95
Parents: 6b54b6e
Author: Julian Hyde <jh...@apache.org>
Authored: Thu Mar 16 11:24:06 2017 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Fri Mar 17 11:48:45 2017 -0700
----------------------------------------------------------------------
.../org/apache/calcite/model/ModelHandler.java | 10 ++--
site/_docs/file_adapter.md | 56 +++++++++++++++++---
2 files changed, 56 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/0e11746a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index bd8bcac..adfdd07 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -236,13 +236,15 @@ public class ModelHandler {
builder.put(extraOperand.camelName, modelUri);
break;
case BASE_DIRECTORY:
+ File f = null;
if (!modelUri.startsWith("inline:")) {
final File file = new File(modelUri);
- final File parentFile = file.getParentFile();
- if (parentFile != null) {
- builder.put(extraOperand.camelName, parentFile);
- }
+ f = file.getParentFile();
}
+ if (f == null) {
+ f = new File("");
+ }
+ builder.put(extraOperand.camelName, f);
break;
case TABLES:
if (jsonSchema instanceof JsonCustomSchema) {
http://git-wip-us.apache.org/repos/asf/calcite/blob/0e11746a/site/_docs/file_adapter.md
----------------------------------------------------------------------
diff --git a/site/_docs/file_adapter.md b/site/_docs/file_adapter.md
index b47d72a..57e9de4 100644
--- a/site/_docs/file_adapter.md
+++ b/site/_docs/file_adapter.md
@@ -22,7 +22,7 @@ limitations under the License.
{% endcomment %}
-->
-# Overview
+## Overview
The file adapter is able to read files in a variety of formats,
and can also read files over various protocols, such as HTTP.
@@ -53,7 +53,11 @@ comprising almost 1/2 of the state's population:
+---------------------+----------------------+
```
-# A simple example
+For simple file formats such as CSV, the file is self-describing and
+you don't even need a model.
+See [CSV files and model-free browsing](#csv_files_and_model_free_browsing).
+
+## A simple example
Let's start with a simple example. First, we need a
[model definition]({{ site.baseurl }}/docs/model.html),
@@ -151,7 +155,7 @@ sqlline> select * from sales.emps;
5 rows selected
{% endhighlight %}
-# Mapping tables
+## Mapping tables
Now for a more complex example. This time we connect to Wikipedia via
HTTP, read pages for US states and cities, and extract data from HTML
@@ -208,7 +212,7 @@ navigation; selectors for both tables and fields follow the
Field definitions may be used to rename or skip source fields, to
select and condition the cell contents and to set a data type.
-# Parsing cell contents
+### Parsing cell contents
The file adapter can select DOM nodes within a cell, replace text
within the selected element, match within the selected text, and
@@ -217,7 +221,7 @@ steps are applied in the order described and replace and match
patterns are based on
[Java regular expressions](http://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html).
-# Further examples
+### Further examples
There are more examples in the form of a script:
@@ -229,7 +233,47 @@ $ ./sqlline -f file/src/test/resources/webjoin.sql
each query containing a join. These are expected and do not affect
query results. These messages will be suppressed in the next release.)
-# Future improvements
+## CSV files and model-free browsing
+
+Some files are describe their own schema, and for these files, we do not need a model. For example, `DEPTS.csv` has an
+integer `DEPTNO` column and a string `NAME` column:
+
+{% highlight json %}
+DEPTNO:int,NAME:string
+10,"Sales"
+20,"Marketing"
+30,"Accounts"
+{% endhighlight %}
+
+You can launch `sqlline`, and pointing the file adapter that directory,
+and every CSV file becomes a table:
+
+{% highlight bash %}
+$ ls file/src/test/resources/sales-csv
+ -rw-r--r-- 1 jhyde jhyde 62 Mar 15 10:16 DEPTS.csv
+ -rw-r--r-- 1 jhyde jhyde 262 Mar 15 10:16 EMPS.csv.gz
+
+$ ./sqlline -u "jdbc:calcite:schemaFactory=org.apache.calcite.adapter.file.FileSchemaFactory;schema.directory=file/src/test/resources/sales-csv"
+sqlline> !tables
++-----------+-------------+------------+------------+
+| TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE |
++-----------+-------------+------------+------------+
+| | adhoc | DEPTS | TABLE |
+| | adhoc | EMPS | TABLE |
++-----------+-------------+------------+------------+
+
+sqlline> select distinct deptno from depts;
++--------+
+| DEPTNO |
++--------+
+| 20 |
+| 10 |
+| 30 |
++--------+
+3 rows selected (0.985 seconds)
+{% endhighlight %}
+
+## Future improvements
We are continuing to enhance the adapter, and would welcome
contributions of new parsing capabilities (for example parsing JSON