You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/05/23 22:14:34 UTC
hive git commit: Revert "HIVE-19615 : Proper handling of is null and
not is null predicate when pushed to Druid (Slim Bougeurra via Ashutosh
Chauhan)"
Repository: hive
Updated Branches:
refs/heads/master 7eadc07ec -> 29e4f3be4
Revert "HIVE-19615 : Proper handling of is null and not is null predicate when pushed to Druid (Slim Bougeurra via Ashutosh Chauhan)"
This reverts commit 7eadc07ec27a259e1ac13d29ff1fb3ef9e0d2226.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/29e4f3be
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/29e4f3be
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/29e4f3be
Branch: refs/heads/master
Commit: 29e4f3be4f2db9b8d37a28e71e8af08073f2d40c
Parents: 7eadc07
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed May 23 15:14:27 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 23 15:14:27 2018 -0700
----------------------------------------------------------------------
.../ql/parse/DruidSqlOperatorConverter.java | 51 +++++++++-----------
.../queries/clientpositive/druidmini_test1.q | 5 --
.../clientpositive/druid/druidmini_test1.q.out | 24 ---------
3 files changed, 24 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/29e4f3be/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java
index ce04eec..2438a86 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DruidSqlOperatorConverter.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -25,7 +25,6 @@ import org.apache.calcite.adapter.druid.DruidExpressions;
import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.adapter.druid.ExtractOperatorConversion;
import org.apache.calcite.adapter.druid.FloorOperatorConversion;
-import org.apache.calcite.adapter.druid.UnarySuffixOperatorConversion;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
@@ -50,46 +49,39 @@ import java.util.Map;
public class DruidSqlOperatorConverter {
private DruidSqlOperatorConverter() {
}
-
private static Map druidOperatorMap = null;
public static final Map<SqlOperator, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter> getDefaultMap() {
if (druidOperatorMap == null) {
- druidOperatorMap = new HashMap<SqlOperator, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter>();
- DruidQuery.DEFAULT_OPERATORS_LIST.stream().forEach(op -> druidOperatorMap.put(op.calciteOperator(), op));
+ druidOperatorMap =
+ new HashMap<SqlOperator, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter>();
+ DruidQuery.DEFAULT_OPERATORS_LIST.stream()
+ .forEach(op -> druidOperatorMap.put(op.calciteOperator(), op));
//Override Hive specific operators
druidOperatorMap.putAll(Maps.asMap(HiveFloorDate.ALL_FUNCTIONS,
- (Function<SqlFunction, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter>) input -> new
- FloorOperatorConversion()
+ (Function<SqlFunction, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter>) input -> new FloorOperatorConversion()
));
druidOperatorMap.putAll(Maps.asMap(HiveExtractDate.ALL_FUNCTIONS,
- (Function<SqlFunction, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter>) input -> new
- ExtractOperatorConversion()
+ (Function<SqlFunction, org.apache.calcite.adapter.druid.DruidSqlOperatorConverter>) input -> new ExtractOperatorConversion()
));
- druidOperatorMap.put(HiveConcat.INSTANCE, new DirectOperatorConversion(HiveConcat.INSTANCE, "concat"));
- druidOperatorMap
- .put(SqlStdOperatorTable.SUBSTRING, new DruidSqlOperatorConverter.DruidSubstringOperatorConversion());
druidOperatorMap
- .put(SqlStdOperatorTable.IS_NULL, new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull"));
- druidOperatorMap.put(SqlStdOperatorTable.IS_NOT_NULL,
- new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull")
+ .put(HiveConcat.INSTANCE, new DirectOperatorConversion(HiveConcat.INSTANCE, "concat"));
+ druidOperatorMap.put(SqlStdOperatorTable.SUBSTRING,
+ new DruidSqlOperatorConverter.DruidSubstringOperatorConversion()
);
}
return druidOperatorMap;
}
- /**
- * Druid operator converter from Hive Substring to Druid SubString.
- * This is a temporary fix that can be removed once we move to a Calcite version including the following.
- * https://issues.apache.org/jira/browse/CALCITE-2226
- */
- public static class DruidSubstringOperatorConversion
- extends org.apache.calcite.adapter.druid.SubstringOperatorConversion {
- @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType, DruidQuery query
+ //@TODO remove this when it is fixed in calcite https://issues.apache.org/jira/browse/HIVE-18996
+ public static class DruidSubstringOperatorConversion extends org.apache.calcite.adapter.druid.SubstringOperatorConversion {
+ @Nullable @Override public String toDruidExpression(RexNode rexNode, RelDataType rowType,
+ DruidQuery query
) {
final RexCall call = (RexCall) rexNode;
- final String arg = DruidExpressions.toDruidExpression(call.getOperands().get(0), rowType, query);
+ final String arg = DruidExpressions.toDruidExpression(
+ call.getOperands().get(0), rowType, query);
if (arg == null) {
return null;
}
@@ -98,7 +90,8 @@ public class DruidSqlOperatorConverter {
final String length;
// SQL is 1-indexed, Druid is 0-indexed.
if (!call.getOperands().get(1).isA(SqlKind.LITERAL)) {
- final String indexExp = DruidExpressions.toDruidExpression(call.getOperands().get(1), rowType, query);
+ final String indexExp = DruidExpressions.toDruidExpression(
+ call.getOperands().get(1), rowType, query);
if (indexExp == null) {
return null;
}
@@ -111,7 +104,8 @@ public class DruidSqlOperatorConverter {
if (call.getOperands().size() > 2) {
//case substring from index with length
if (!call.getOperands().get(2).isA(SqlKind.LITERAL)) {
- length = DruidExpressions.toDruidExpression(call.getOperands().get(2), rowType, query);
+ length = DruidExpressions.toDruidExpression(
+ call.getOperands().get(2), rowType, query);
if (length == null) {
return null;
}
@@ -123,7 +117,10 @@ public class DruidSqlOperatorConverter {
//case substring from index to the end
length = DruidExpressions.numberLiteral(-1);
}
- return DruidQuery.format("substring(%s, %s, %s)", arg, indexStart, length);
+ return DruidQuery.format("substring(%s, %s, %s)",
+ arg,
+ indexStart,
+ length);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/29e4f3be/ql/src/test/queries/clientpositive/druidmini_test1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_test1.q b/ql/src/test/queries/clientpositive/druidmini_test1.q
index d18301e..4a1bdc5 100644
--- a/ql/src/test/queries/clientpositive/druidmini_test1.q
+++ b/ql/src/test/queries/clientpositive/druidmini_test1.q
@@ -120,8 +120,3 @@ SELECT `__time`
FROM druid_table_n3
WHERE (`__time` BETWEEN '1968-01-01 00:00:00' AND '1970-01-01 00:00:00')
OR (`__time` BETWEEN '1968-02-01 00:00:00' AND '1970-04-01 00:00:00') ORDER BY `__time` ASC LIMIT 10;
-
--- Running this against Druid will if Druid version does not include
--- this patch https://github.com/druid-io/druid/commit/219e77aeac9b07dc20dd9ab2dd537f3f17498346
-
-explain select (cstring1 is null ) AS is_null, (cint is not null ) as isnotnull FROM druid_table;
http://git-wip-us.apache.org/repos/asf/hive/blob/29e4f3be/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
index 16ef603..a878443 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_test1.q.out
@@ -797,27 +797,3 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
1969-12-31 15:59:00.0 US/Pacific
1969-12-31 15:59:00.0 US/Pacific
1969-12-31 15:59:00.0 US/Pacific
-PREHOOK: query: explain select (cstring1 is null ) AS is_null, (cint is not null ) as isnotnull FROM druid_table
-PREHOOK: type: QUERY
-POSTHOOK: query: explain select (cstring1 is null ) AS is_null, (cint is not null ) as isnotnull FROM druid_table
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-0 is a root stage
-
-STAGE PLANS:
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- TableScan
- alias: druid_table
- properties:
- druid.fieldNames vc,vc0
- druid.fieldTypes boolean,boolean
- druid.query.json {"queryType":"scan","dataSource":"default.druid_table","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"(\"cstring1\" isnull)","outputType":"FLOAT"},{"type":"expression","name":"vc0","expression":"(\"cint\" notnull)","outputType":"FLOAT"}],"columns":["vc","vc0"],"resultFormat":"compactedList"}
- druid.query.type scan
- Select Operator
- expressions: vc (type: boolean), vc0 (type: boolean)
- outputColumnNames: _col0, _col1
- ListSink
-