You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/08 20:42:52 UTC
[02/51] [partial] hive git commit: Revert "HIVE-14671 : merge master
into hive-14535 (Wei Zheng)"
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
index d7a83f7..a231543 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
@@ -69,16 +68,10 @@ public class GenMRFileSink1 implements NodeProcessor {
GenMRProcContext ctx = (GenMRProcContext) opProcCtx;
ParseContext parseCtx = ctx.getParseCtx();
boolean chDir = false;
- // we should look take the parent of fsOp's task as the current task.
- FileSinkOperator fsOp = (FileSinkOperator) nd;
- Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx
- .getMapCurrCtx();
- GenMapRedCtx mapredCtx = mapCurrCtx.get(fsOp.getParentOperators().get(0));
- Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
-
- ctx.setCurrTask(currTask);
+ Task<? extends Serializable> currTask = ctx.getCurrTask();
ctx.addRootIfPossible(currTask);
+ FileSinkOperator fsOp = (FileSinkOperator) nd;
boolean isInsertTable = // is INSERT OVERWRITE TABLE
GenMapRedUtils.isInsertInto(parseCtx, fsOp);
HiveConf hconf = parseCtx.getConf();
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 87fff3e..38157a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -131,7 +131,11 @@ import com.google.common.collect.Interner;
* map-reduce tasks.
*/
public final class GenMapRedUtils {
- private static final Logger LOG = LoggerFactory.getLogger("org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils");
+ private static Logger LOG;
+
+ static {
+ LOG = LoggerFactory.getLogger("org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils");
+ }
public static boolean needsTagging(ReduceWork rWork) {
return rWork != null && (rWork.getReducer().getClass() == JoinOperator.class ||
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
index 85d46f3..b2893e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.parse.GenMapRedWalker;
+import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -79,6 +80,8 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import com.clearspring.analytics.util.Lists;
+
/**
* Implementation of one of the rule-based map join optimization. User passes hints to specify
* map-joins and during this optimization, all user specified map joins are converted to MapJoins -
@@ -431,8 +434,7 @@ public class MapJoinProcessor extends Transform {
smbJoinDesc.getValueTblDescs(), smbJoinDesc.getValueTblDescs(),
smbJoinDesc.getOutputColumnNames(),
bigTablePos, smbJoinDesc.getConds(),
- smbJoinDesc.getFilters(), smbJoinDesc.isNoOuterJoin(), smbJoinDesc.getDumpFilePrefix(),
- smbJoinDesc.getNoConditionalTaskSize());
+ smbJoinDesc.getFilters(), smbJoinDesc.isNoOuterJoin(), smbJoinDesc.getDumpFilePrefix());
mapJoinDesc.setStatistics(smbJoinDesc.getStatistics());
@@ -1185,7 +1187,7 @@ public class MapJoinProcessor extends Transform {
MapJoinDesc mapJoinDescriptor =
new MapJoinDesc(keyExprMap, keyTableDesc, newValueExprs, valueTableDescs,
valueFilteredTableDescs, outputColumnNames, mapJoinPos, joinCondns, filters, op
- .getConf().getNoOuterJoin(), dumpFilePrefix, op.getConf().getNoConditionalTaskSize());
+ .getConf().getNoOuterJoin(), dumpFilePrefix);
mapJoinDescriptor.setStatistics(op.getConf().getStatistics());
mapJoinDescriptor.setTagOrder(tagOrder);
mapJoinDescriptor.setNullSafes(desc.getNullSafes());
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 92225ac..a3a19f4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -210,7 +210,7 @@ public class Optimizer {
if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCORRELATION) &&
!HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEGROUPBYSKEW) &&
!HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME) &&
- !isTezExecEngine && !isSparkExecEngine) {
+ !isTezExecEngine) {
transformations.add(new CorrelationOptimizer());
}
if (HiveConf.getFloatVar(hiveConf, HiveConf.ConfVars.HIVELIMITPUSHDOWNMEMORYUSAGE) > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
index b51af55..60a8604 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SetReducerParallelism.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.stats.StatsUtils;
import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.AUTOPARALLEL;
import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.UNIFORM;
-import static org.apache.hadoop.hive.ql.plan.ReduceSinkDesc.ReducerTraits.FIXED;
/**
* SetReducerParallelism determines how many reducers should
@@ -107,7 +106,6 @@ public class SetReducerParallelism implements NodeProcessor {
}
} else {
LOG.info("Number of reducers determined to be: "+desc.getNumReducers());
- desc.setReducerTraits(EnumSet.of(FIXED)); // usually controlled by bucketing
}
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
index 8c43774..5b2c9c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
@@ -31,11 +31,10 @@ public class CalciteSemanticException extends SemanticException {
public enum UnsupportedFeature {
Distinct_without_an_aggreggation, Duplicates_in_RR, Filter_expression_with_non_boolean_return_type,
- Having_clause_without_any_groupby, Invalid_column_reference, Invalid_decimal,
+ Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal,
Less_than_equal_greater_than, Others, Same_name_in_multiple_expressions,
Schema_less_table, Select_alias_in_having_clause, Select_transform, Subquery,
- Table_sample_clauses, UDTF, Union_type, Unique_join,
- HighPrecissionTimestamp // CALCITE-1690
+ Table_sample_clauses, UDTF, Union_type, Unique_join
};
private UnsupportedFeature unsupportedFeature;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index e339d0a..6ccd879 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -48,7 +48,6 @@ import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexLocalRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
-import org.apache.calcite.rex.RexPatternFieldRef;
import org.apache.calcite.rex.RexRangeRef;
import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.rex.RexUtil;
@@ -1075,11 +1074,6 @@ public class HiveCalciteUtil {
// it seems that it is not used by anything.
return false;
}
-
- @Override
- public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
- return false;
- }
}
public static Set<Integer> getInputRefs(RexNode expr) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HivePlannerContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HivePlannerContext.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HivePlannerContext.java
index d0b1757..9a65de3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HivePlannerContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HivePlannerContext.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite;
-import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.plan.Context;
import org.apache.calcite.rel.RelNode;
import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveAlgorithmsConf;
@@ -28,14 +27,11 @@ import java.util.Set;
public class HivePlannerContext implements Context {
private HiveAlgorithmsConf algoConfig;
private HiveRulesRegistry registry;
- private CalciteConnectionConfig calciteConfig;
private Set<RelNode> corrScalarRexSQWithAgg;
- public HivePlannerContext(HiveAlgorithmsConf algoConfig, HiveRulesRegistry registry,
- CalciteConnectionConfig calciteConfig, Set<RelNode> corrScalarRexSQWithAgg) {
+ public HivePlannerContext(HiveAlgorithmsConf algoConfig, HiveRulesRegistry registry, Set<RelNode> corrScalarRexSQWithAgg) {
this.algoConfig = algoConfig;
this.registry = registry;
- this.calciteConfig = calciteConfig;
// this is to keep track if a subquery is correlated and contains aggregate
// this is computed in CalcitePlanner while planning and is later required by subuery remove rule
// hence this is passed using HivePlannerContext
@@ -49,9 +45,6 @@ public class HivePlannerContext implements Context {
if (clazz.isInstance(registry)) {
return clazz.cast(registry);
}
- if (clazz.isInstance(calciteConfig)) {
- return clazz.cast(calciteConfig);
- }
if(clazz.isInstance(corrScalarRexSQWithAgg)) {
return clazz.cast(corrScalarRexSQWithAgg);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java
index 8e52d88..2aadf50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java
@@ -27,7 +27,6 @@ import org.apache.calcite.rel.logical.LogicalExchange;
import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.logical.LogicalIntersect;
import org.apache.calcite.rel.logical.LogicalJoin;
-import org.apache.calcite.rel.logical.LogicalMatch;
import org.apache.calcite.rel.logical.LogicalMinus;
import org.apache.calcite.rel.logical.LogicalProject;
import org.apache.calcite.rel.logical.LogicalSort;
@@ -141,10 +140,6 @@ public class HiveRelShuttleImpl implements HiveRelShuttle {
public RelNode visit(RelNode other) {
return visitChildren(other);
}
-
- public RelNode visit(LogicalMatch match) {
- return visitChildren(match);
- }
}
// End RelShuttleImpl.java
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveExtractDate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveExtractDate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveExtractDate.java
index 0b94b8a..4edc4df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveExtractDate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveExtractDate.java
@@ -24,7 +24,6 @@ import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
-import org.apache.calcite.sql.type.SqlTypeTransforms;
import com.google.common.collect.Sets;
@@ -43,10 +42,9 @@ public class HiveExtractDate extends SqlFunction {
Sets.newHashSet(YEAR, QUARTER, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND);
private HiveExtractDate(String name) {
- super(name, SqlKind.EXTRACT,
- ReturnTypes.cascade(ReturnTypes.INTEGER, SqlTypeTransforms.FORCE_NULLABLE), null,
- OperandTypes.INTERVALINTERVAL_INTERVALDATETIME,
- SqlFunctionCategory.SYSTEM);
+ super(name, SqlKind.EXTRACT, ReturnTypes.INTEGER_NULLABLE, null,
+ OperandTypes.INTERVALINTERVAL_INTERVALDATETIME,
+ SqlFunctionCategory.SYSTEM);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java
deleted file mode 100644
index dd39056..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveDruidProjectFilterTransposeRule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
-
-import org.apache.calcite.adapter.druid.DruidQuery;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.rules.ProjectFilterTransposeRule;
-import org.apache.calcite.rel.rules.PushProjector;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
-
-/**
- * Rule to push a {@link org.apache.calcite.rel.core.Project}
- * past a {@link org.apache.calcite.rel.core.Filter}
- * when {@code Filter} is on top of a {@link DruidQuery}.
- * TODO: Replace this class with calcite DruidRules.DruidFilterProjectTransposeRule
- * once we upgrade to calcite 1.13
- */
-public class HiveDruidProjectFilterTransposeRule
- extends ProjectFilterTransposeRule {
-
- public static final HiveDruidProjectFilterTransposeRule INSTANCE =
- new HiveDruidProjectFilterTransposeRule();
-
- private HiveDruidProjectFilterTransposeRule() {
- super(
- operand(Project.class,
- operand(Filter.class, operand(DruidQuery.class, none()))),
- PushProjector.ExprCondition.FALSE,
- HiveRelFactories.HIVE_BUILDER);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
index a4da6db..4b8568e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
@@ -106,6 +106,57 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule {
}
}
+ /*
+ * Any predicates pushed down to joinFilters that aren't equality conditions:
+ * put them back as aboveFilters because Hive doesn't support not equi join
+ * conditions.
+ */
+ @Override
+ protected void validateJoinFilters(List<RexNode> aboveFilters, List<RexNode> joinFilters,
+ Join join, JoinRelType joinType) {
+ if (joinType.equals(JoinRelType.INNER)) {
+ ListIterator<RexNode> filterIter = joinFilters.listIterator();
+ while (filterIter.hasNext()) {
+ RexNode exp = filterIter.next();
+
+ if (exp instanceof RexCall) {
+ RexCall c = (RexCall) exp;
+ boolean validHiveJoinFilter = false;
+
+ if ((c.getOperator().getKind() == SqlKind.EQUALS)) {
+ validHiveJoinFilter = true;
+ for (RexNode rn : c.getOperands()) {
+ // NOTE: Hive dis-allows projections from both left & right side
+ // of join condition. Example: Hive disallows
+ // (r1.x +r2.x)=(r1.y+r2.y) on join condition.
+ if (filterRefersToBothSidesOfJoin(rn, join)) {
+ validHiveJoinFilter = false;
+ break;
+ }
+ }
+ } else if ((c.getOperator().getKind() == SqlKind.LESS_THAN)
+ || (c.getOperator().getKind() == SqlKind.GREATER_THAN)
+ || (c.getOperator().getKind() == SqlKind.LESS_THAN_OR_EQUAL)
+ || (c.getOperator().getKind() == SqlKind.GREATER_THAN_OR_EQUAL)) {
+ validHiveJoinFilter = true;
+ // NOTE: Hive dis-allows projections from both left & right side of
+ // join in in equality condition. Example: Hive disallows (r1.x <
+ // r2.x) on join condition.
+ if (filterRefersToBothSidesOfJoin(c, join)) {
+ validHiveJoinFilter = false;
+ }
+ }
+
+ if (validHiveJoinFilter)
+ continue;
+ }
+
+ aboveFilters.add(exp);
+ filterIter.remove();
+ }
+ }
+ }
+
private boolean filterRefersToBothSidesOfJoin(RexNode filter, Join j) {
boolean refersToBothSides = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
index f3d7293..4cfe782 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
@@ -55,89 +55,36 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
-import org.apache.calcite.plan.RelOptRuleOperand;
-import org.apache.calcite.rel.AbstractRelNode;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.JoinRelType;
-
-
-public abstract class HivePointLookupOptimizerRule extends RelOptRule {
/**
- * This optimization will take a Filter or expression, and if its predicate contains
+ * This optimization will take a Filter expression, and if its predicate contains
* an OR operator whose children are constant equality expressions, it will try
* to generate an IN clause (which is more efficient). If the OR operator contains
* AND operator children, the optimization might generate an IN clause that uses
* structs.
*/
- public static class FilterCondition extends HivePointLookupOptimizerRule {
- public FilterCondition (int minNumORClauses) {
- super(operand(Filter.class, any()), minNumORClauses);
- }
-
- public void onMatch(RelOptRuleCall call) {
- final Filter filter = call.rel(0);
- final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
- final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
- analyzeCondition(call , rexBuilder, filter, condition);
- }
-
- @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
- final Filter filter = (Filter) node;
- return filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
- }
- }
-
-/**
- * This optimization will take a Join or expression, and if its join condition contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
- */
- public static class JoinCondition extends HivePointLookupOptimizerRule {
- public JoinCondition (int minNumORClauses) {
- super(operand(Join.class, any()), minNumORClauses);
- }
-
- public void onMatch(RelOptRuleCall call) {
- final Join join = call.rel(0);
- final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
- final RexNode condition = RexUtil.pullFactors(rexBuilder, join.getCondition());
- analyzeCondition(call , rexBuilder, join, condition);
- }
-
- @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
- final Join join = (Join) node;
- return join.copy(join.getTraitSet(),
- newCondition,
- join.getLeft(),
- join.getRight(),
- join.getJoinType(),
- join.isSemiJoinDone());
- }
- }
+public class HivePointLookupOptimizerRule extends RelOptRule {
protected static final Log LOG = LogFactory.getLog(HivePointLookupOptimizerRule.class);
- // Minimum number of OR clauses needed to transform into IN clauses
- protected final int minNumORClauses;
- protected abstract RelNode copyNode(AbstractRelNode node, RexNode newCondition);
+ // Minimum number of OR clauses needed to transform into IN clauses
+ private final int minNumORClauses;
- protected HivePointLookupOptimizerRule(
- RelOptRuleOperand operand, int minNumORClauses) {
- super(operand);
+ public HivePointLookupOptimizerRule(int minNumORClauses) {
+ super(operand(Filter.class, any()));
this.minNumORClauses = minNumORClauses;
}
- public void analyzeCondition(RelOptRuleCall call,
- RexBuilder rexBuilder,
- AbstractRelNode node,
- RexNode condition) {
+ public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+
+ final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
+
+ final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
// 1. We try to transform possible candidates
- RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, node,
+ RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, filter,
minNumORClauses);
RexNode newCondition = transformIntoInClause.apply(condition);
@@ -150,10 +97,10 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
return;
}
- // 4. We create the Filter/Join with the new condition
- RelNode newNode = copyNode(node, newCondition);
+ // 4. We create the filter with the new condition
+ RelNode newFilter = filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
- call.transformTo(newNode);
+ call.transformTo(newFilter);
}
@@ -162,11 +109,11 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
*/
protected static class RexTransformIntoInClause extends RexShuttle {
private final RexBuilder rexBuilder;
- private final AbstractRelNode nodeOp;
+ private final Filter filterOp;
private final int minNumORClauses;
- RexTransformIntoInClause(RexBuilder rexBuilder, AbstractRelNode nodeOp, int minNumORClauses) {
- this.nodeOp = nodeOp;
+ RexTransformIntoInClause(RexBuilder rexBuilder, Filter filterOp, int minNumORClauses) {
+ this.filterOp = filterOp;
this.rexBuilder = rexBuilder;
this.minNumORClauses = minNumORClauses;
}
@@ -182,7 +129,7 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
if (operand.getKind() == SqlKind.OR) {
try {
newOperand = transformIntoInClauseCondition(rexBuilder,
- nodeOp.getRowType(), operand, minNumORClauses);
+ filterOp.getRowType(), operand, minNumORClauses);
if (newOperand == null) {
newOperand = operand;
}
@@ -200,7 +147,7 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
case OR:
try {
node = transformIntoInClauseCondition(rexBuilder,
- nodeOp.getRowType(), call, minNumORClauses);
+ filterOp.getRowType(), call, minNumORClauses);
if (node == null) {
return call;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
index 1487ed4..fd19d99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortTransposeRule.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
-import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
@@ -26,12 +25,7 @@ import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexCallBinding;
-import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.validate.SqlMonotonicity;
import org.apache.calcite.util.mapping.Mappings;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
@@ -65,28 +59,16 @@ public class HiveProjectSortTransposeRule extends RelOptRule {
public void onMatch(RelOptRuleCall call) {
final HiveProject project = call.rel(0);
final HiveSortLimit sort = call.rel(1);
- final RelOptCluster cluster = project.getCluster();
// Determine mapping between project input and output fields. If sort
// relies on non-trivial expressions, we can't push.
final Mappings.TargetMapping map =
- RelOptUtil.permutationIgnoreCast(
+ RelOptUtil.permutation(
project.getProjects(), project.getInput().getRowType()).inverse();
for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
if (map.getTarget(fc.getFieldIndex()) < 0) {
return;
}
- final RexNode node = project.getProjects().get(map.getTarget(fc.getFieldIndex()));
- if (node.isA(SqlKind.CAST)) {
- // Check whether it is a monotonic preserving cast, otherwise we cannot push
- final RexCall cast = (RexCall) node;
- final RexCallBinding binding =
- RexCallBinding.create(cluster.getTypeFactory(), cast,
- ImmutableList.of(RexUtil.apply(map, sort.getCollation())));
- if (cast.getOperator().getMonotonicity(binding) == SqlMonotonicity.NOT_MONOTONIC) {
- return;
- }
- }
}
// Create new collation
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
index d1be4bc..fe29850 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSortProjectTransposeRule.java
@@ -17,13 +17,23 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.rules.SortProjectTransposeRule;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.util.mapping.Mappings;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
-public class HiveSortProjectTransposeRule extends SortProjectTransposeRule {
+import com.google.common.collect.ImmutableList;
+
+public class HiveSortProjectTransposeRule extends RelOptRule {
public static final HiveSortProjectTransposeRule INSTANCE =
new HiveSortProjectTransposeRule();
@@ -40,6 +50,10 @@ public class HiveSortProjectTransposeRule extends SortProjectTransposeRule {
operand(HiveProject.class, any())));
}
+ protected HiveSortProjectTransposeRule(RelOptRuleOperand operand) {
+ super(operand);
+ }
+
//~ Methods ----------------------------------------------------------------
@Override
@@ -54,4 +68,34 @@ public class HiveSortProjectTransposeRule extends SortProjectTransposeRule {
return true;
}
+ // implement RelOptRule
+ public void onMatch(RelOptRuleCall call) {
+ final HiveSortLimit sort = call.rel(0);
+ final HiveProject project = call.rel(1);
+
+ // Determine mapping between project input and output fields. If sort
+ // relies on non-trivial expressions, we can't push.
+ final Mappings.TargetMapping map =
+ RelOptUtil.permutation(
+ project.getProjects(), project.getInput().getRowType());
+ for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
+ if (map.getTargetOpt(fc.getFieldIndex()) < 0) {
+ return;
+ }
+ }
+
+ // Create new collation
+ final RelCollation newCollation =
+ RelCollationTraitDef.INSTANCE.canonize(
+ RexUtil.apply(map, sort.getCollation()));
+
+ // New operators
+ final HiveSortLimit newSort = sort.copy(sort.getTraitSet().replace(newCollation),
+ project.getInput(), newCollation, sort.offset, sort.fetch);
+ final RelNode newProject = project.copy(sort.getTraitSet(),
+ ImmutableList.<RelNode>of(newSort));
+
+ call.transformTo(newProject);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java
index c692cc0..76e0780 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java
@@ -29,7 +29,6 @@ import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rex.LogicVisitor;
import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexSubQuery;
@@ -44,12 +43,10 @@ import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.Pair;
-import org.apache.calcite.util.Util;
-import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
-import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -75,65 +72,64 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
*/
public abstract class HiveSubQueryRemoveRule extends RelOptRule{
- public static final HiveSubQueryRemoveRule REL_NODE =
- new HiveSubQueryRemoveRule(
- operand(RelNode.class, null, HiveSubQueryFinder.RELNODE_PREDICATE,
- any()),
- HiveRelFactories.HIVE_BUILDER, "SubQueryRemoveRule:Filter") {
- public void onMatch(RelOptRuleCall call) {
- final RelNode relNode = call.rel(0);
- //TODO: replace HiveSubQRemoveRelBuilder with calcite's once calcite 1.11.0 is released
- final HiveSubQRemoveRelBuilder builder = new HiveSubQRemoveRelBuilder(null, call.rel(0).getCluster(), null);
-
- // if subquery is in FILTER
- if(relNode instanceof Filter) {
- final Filter filter = call.rel(0);
+ public static final HiveSubQueryRemoveRule PROJECT =
+ new HiveSubQueryRemoveRule(
+ operand(Project.class, null, RexUtil.SubQueryFinder.PROJECT_PREDICATE,
+ any()),
+ HiveRelFactories.HIVE_BUILDER, "SubQueryRemoveRule:Project") {
+ public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ //TODO: replace HiveSubQRemoveRelBuilder with calcite's once calcite 1.11.0 is released
+ final HiveSubQRemoveRelBuilder builder = new HiveSubQRemoveRelBuilder(null, call.rel(0).getCluster(), null);
final RexSubQuery e =
- RexUtil.SubQueryFinder.find(filter.getCondition());
+ RexUtil.SubQueryFinder.find(project.getProjects());
assert e != null;
-
final RelOptUtil.Logic logic =
- LogicVisitor.find(RelOptUtil.Logic.TRUE,
- ImmutableList.of(filter.getCondition()), e);
- builder.push(filter.getInput());
+ LogicVisitor.find(RelOptUtil.Logic.TRUE_FALSE_UNKNOWN,
+ project.getProjects(), e);
+ builder.push(project.getInput());
final int fieldCount = builder.peek().getRowType().getFieldCount();
-
- assert(filter instanceof HiveFilter);
- Set<RelNode> corrScalarQueries = filter.getCluster().getPlanner().getContext().unwrap(Set.class);
- boolean isCorrScalarQuery = corrScalarQueries.contains(e.rel);
-
- final RexNode target = apply(e, HiveFilter.getVariablesSet(e), logic,
- builder, 1, fieldCount, isCorrScalarQuery);
+ final RexNode target = apply(e, HiveFilter.getVariablesSet(e),
+ logic, builder, 1, fieldCount, false);
final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target);
- builder.filter(shuttle.apply(filter.getCondition()));
- builder.project(fields(builder, filter.getRowType().getFieldCount()));
+ builder.project(shuttle.apply(project.getProjects()),
+ project.getRowType().getFieldNames());
call.transformTo(builder.build());
}
- // if subquery is in PROJECT
- else if(relNode instanceof Project) {
- final Project project = call.rel(0);
+ };
+
+ public static final HiveSubQueryRemoveRule FILTER =
+ new HiveSubQueryRemoveRule(
+ operand(Filter.class, null, RexUtil.SubQueryFinder.FILTER_PREDICATE,
+ any()),
+ HiveRelFactories.HIVE_BUILDER, "SubQueryRemoveRule:Filter") {
+ public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+ //final RelBuilder builder = call.builder();
+ //TODO: replace HiveSubQRemoveRelBuilder with calcite's once calcite 1.11.0 is released
+ final HiveSubQRemoveRelBuilder builder = new HiveSubQRemoveRelBuilder(null, call.rel(0).getCluster(), null);
final RexSubQuery e =
- RexUtil.SubQueryFinder.find(project.getProjects());
+ RexUtil.SubQueryFinder.find(filter.getCondition());
assert e != null;
final RelOptUtil.Logic logic =
- LogicVisitor.find(RelOptUtil.Logic.TRUE_FALSE_UNKNOWN,
- project.getProjects(), e);
- builder.push(project.getInput());
+ LogicVisitor.find(RelOptUtil.Logic.TRUE,
+ ImmutableList.of(filter.getCondition()), e);
+ builder.push(filter.getInput());
final int fieldCount = builder.peek().getRowType().getFieldCount();
- Set<RelNode> corrScalarQueries = project.getCluster().getPlanner().getContext().unwrap(Set.class);
+ assert(filter instanceof HiveFilter);
+ Set<RelNode> corrScalarQueries = filter.getCluster().getPlanner().getContext().unwrap(Set.class);
boolean isCorrScalarQuery = corrScalarQueries.contains(e.rel);
- final RexNode target = apply(e, HiveFilter.getVariablesSet(e),
- logic, builder, 1, fieldCount, isCorrScalarQuery);
+ final RexNode target = apply(e, HiveFilter.getVariablesSet(e), logic,
+ builder, 1, fieldCount, isCorrScalarQuery);
final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target);
- builder.project(shuttle.apply(project.getProjects()),
- project.getRowType().getFieldNames());
+ builder.filter(shuttle.apply(filter.getCondition()));
+ builder.project(fields(builder, filter.getRowType().getFieldCount()));
call.transformTo(builder.build());
}
- }
- };
+ };
private HiveSubQueryRemoveRule(RelOptRuleOperand operand,
RelBuilderFactory relBuilderFactory,
@@ -168,25 +164,6 @@ public abstract class HiveSubQueryRemoveRule extends RelOptRule{
boolean isCorrScalarAgg) {
switch (e.getKind()) {
case SCALAR_QUERY:
- builder.push(e.rel);
- // returns single row/column
- builder.aggregate(builder.groupKey(),
- builder.count(false, "cnt"));
-
- SqlFunction countCheck = new SqlFunction("sq_count_check", SqlKind.OTHER_FUNCTION, ReturnTypes.BIGINT,
- InferTypes.RETURN_TYPE, OperandTypes.NUMERIC, SqlFunctionCategory.USER_DEFINED_FUNCTION);
-
- // we create FILTER (sq_count_check(count()) <= 1) instead of PROJECT because RelFieldTrimmer
- // ends up getting rid of Project since it is not used further up the tree
- builder.filter(builder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
- builder.call(countCheck, builder.field("cnt")),
- builder.literal(1)));
- if( !variablesSet.isEmpty())
- {
- builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
- }
- else
- builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
if(isCorrScalarAgg) {
// Transformation :
// Outer Query Left Join (inner query) on correlated predicate and preserve rows only from left side.
@@ -204,9 +181,7 @@ public abstract class HiveSubQueryRemoveRule extends RelOptRule{
final ImmutableList.Builder<RexNode> operands = ImmutableList.builder();
RexNode literal;
if(isAggZeroOnEmpty(e)) {
- // since count has a return type of BIG INT we need to make a literal of type big int
- // relbuilder's literal doesn't allow this
- literal = e.rel.getCluster().getRexBuilder().makeBigintLiteral(new BigDecimal(0));
+ literal = builder.literal(0);
}
else {
literal = e.rel.getCluster().getRexBuilder().makeNullLiteral(getAggTypeForScalarSub(e));
@@ -218,7 +193,26 @@ public abstract class HiveSubQueryRemoveRule extends RelOptRule{
//Transformation is to left join for correlated predicates and inner join otherwise,
// but do a count on inner side before that to make sure it generates atmost 1 row.
+ builder.push(e.rel);
+ // returns single row/column
+ builder.aggregate(builder.groupKey(),
+ builder.count(false, "cnt"));
+
+ SqlFunction countCheck = new SqlFunction("sq_count_check", SqlKind.OTHER_FUNCTION, ReturnTypes.BIGINT,
+ InferTypes.RETURN_TYPE, OperandTypes.NUMERIC, SqlFunctionCategory.USER_DEFINED_FUNCTION);
+ // we create FILTER (sq_count_check(count()) <= 1) instead of PROJECT because RelFieldTrimmer
+ // ends up getting rid of Project since it is not used further up the tree
+ builder.filter(builder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
+ builder.call(countCheck, builder.field("cnt")),
+ builder.literal(1)));
+
+ if( !variablesSet.isEmpty())
+ {
+ builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
+ }
+ else
+ builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
builder.push(e.rel);
builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
offset++;
@@ -461,72 +455,6 @@ public abstract class HiveSubQueryRemoveRule extends RelOptRule{
return RexUtil.eq(subQuery, this.subQuery) ? replacement : subQuery;
}
}
-
- // TODO:
- // Following HiveSubQueryFinder has been copied from RexUtil::SubQueryFinder
- // since there is BUG in there (CALCITE-1726).
- // Once CALCITE-1726 is fixed we should get rid of the following code
- /** Visitor that throws {@link org.apache.calcite.util.Util.FoundOne} if
- * applied to an expression that contains a {@link RexSubQuery}. */
- public static class HiveSubQueryFinder extends RexVisitorImpl<Void> {
- public static final HiveSubQueryFinder INSTANCE = new HiveSubQueryFinder();
-
- /** Returns whether a {@link Project} contains a sub-query. */
- public static final Predicate<RelNode> RELNODE_PREDICATE=
- new Predicate<RelNode>() {
- public boolean apply(RelNode relNode) {
- if (relNode instanceof Project) {
- Project project = (Project)relNode;
- for (RexNode node : project.getProjects()) {
- try {
- node.accept(INSTANCE);
- } catch (Util.FoundOne e) {
- return true;
- }
- }
- return false;
- }
- else if (relNode instanceof Filter) {
- try {
- ((Filter)relNode).getCondition().accept(INSTANCE);
- return false;
- } catch (Util.FoundOne e) {
- return true;
- }
- }
- return false;
- }
- };
-
- private HiveSubQueryFinder() {
- super(true);
- }
-
- @Override public Void visitSubQuery(RexSubQuery subQuery) {
- throw new Util.FoundOne(subQuery);
- }
-
- public static RexSubQuery find(Iterable<RexNode> nodes) {
- for (RexNode node : nodes) {
- try {
- node.accept(INSTANCE);
- } catch (Util.FoundOne e) {
- return (RexSubQuery) e.getNode();
- }
- }
- return null;
- }
-
- public static RexSubQuery find(RexNode node) {
- try {
- node.accept(INSTANCE);
- return null;
- } catch (Util.FoundOne e) {
- return (RexSubQuery) e.getNode();
- }
- }
- }
-
}
// End SubQueryRemoveRule.java
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java
index 81de33f..38d7906 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveMaterializedViewFilterScanRule.java
@@ -21,7 +21,6 @@ import java.util.Collections;
import java.util.List;
import org.apache.calcite.plan.RelOptMaterialization;
-import org.apache.calcite.plan.RelOptMaterializations;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
@@ -78,7 +77,7 @@ public class HiveMaterializedViewFilterScanRule extends RelOptRule {
// Costing is done in transformTo(), so we call it repeatedly with all applicable
// materialized views and cheapest one will be picked
List<RelOptMaterialization> applicableMaterializations =
- RelOptMaterializations.getApplicableMaterializations(root, materializations);
+ VolcanoPlanner.getApplicableMaterializations(root, materializations);
for (RelOptMaterialization materialization : applicableMaterializations) {
List<RelNode> subs = new MaterializedViewSubstitutionVisitor(
materialization.queryRel, root, relBuilderFactory).go(materialization.tableRel);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
index a25b58b..6f26d7d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
@@ -29,7 +29,6 @@ import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlKind;
@@ -302,15 +301,4 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
return op;
}
-
- public Double visitLiteral(RexLiteral literal) {
- if (literal.isAlwaysFalse()) {
- return 0.0;
- } else if (literal.isAlwaysTrue()) {
- return 1.0;
- } else {
- assert false;
- }
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
index 9bcdd0c..69e157e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
@@ -165,7 +165,7 @@ public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Pred
rexBuilder.makeInputRef(project, expr.i), expr.e));
}
}
- return RelOptPredicateList.of(rexBuilder, projectPullUpPredicates);
+ return RelOptPredicateList.of(projectPullUpPredicates);
}
/** Infers predicates for a {@link org.apache.calcite.rel.core.Join}. */
@@ -202,7 +202,6 @@ public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Pred
final RelNode input = agg.getInput();
final RelOptPredicateList inputInfo = mq.getPulledUpPredicates(input);
final List<RexNode> aggPullUpPredicates = new ArrayList<>();
- final RexBuilder rexBuilder = agg.getCluster().getRexBuilder();
ImmutableBitSet groupKeys = agg.getGroupSet();
Mapping m = Mappings.create(MappingType.PARTIAL_FUNCTION,
@@ -220,7 +219,7 @@ public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Pred
aggPullUpPredicates.add(r);
}
}
- return RelOptPredicateList.of(rexBuilder, aggPullUpPredicates);
+ return RelOptPredicateList.of(aggPullUpPredicates);
}
/**
@@ -272,7 +271,7 @@ public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Pred
if (!disjPred.isAlwaysTrue()) {
preds.add(disjPred);
}
- return RelOptPredicateList.of(rB, preds);
+ return RelOptPredicateList.of(preds);
}
/**
@@ -412,7 +411,6 @@ public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Pred
final JoinRelType joinType = joinRel.getJoinType();
final List<RexNode> leftPreds = ImmutableList.copyOf(RelOptUtil.conjunctions(leftChildPredicates));
final List<RexNode> rightPreds = ImmutableList.copyOf(RelOptUtil.conjunctions(rightChildPredicates));
- final RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
switch (joinType) {
case INNER:
case LEFT:
@@ -478,13 +476,13 @@ public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Pred
pulledUpPredicates = Iterables.concat(leftPreds, rightPreds,
RelOptUtil.conjunctions(joinRel.getCondition()), inferredPredicates);
}
- return RelOptPredicateList.of(rexBuilder,
+ return RelOptPredicateList.of(
pulledUpPredicates, leftInferredPredicates, rightInferredPredicates);
case LEFT:
- return RelOptPredicateList.of(rexBuilder,
+ return RelOptPredicateList.of(
leftPreds, EMPTY_LIST, rightInferredPredicates);
case RIGHT:
- return RelOptPredicateList.of(rexBuilder,
+ return RelOptPredicateList.of(
rightPreds, leftInferredPredicates, EMPTY_LIST);
default:
assert inferredPredicates.size() == 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
index a43d2be..0dc0c24 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
@@ -37,8 +37,6 @@ import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
public class ASTBuilder {
@@ -271,23 +269,19 @@ public class ASTBuilder {
type = ((Boolean) val).booleanValue() ? HiveParser.KW_TRUE : HiveParser.KW_FALSE;
break;
case DATE: {
- //Calcite Calendar is always GMT, Hive atm uses JVM local
- final Calendar c = (Calendar) literal.getValue();
- final DateTime dt = new DateTime(c.getTimeInMillis(), DateTimeZone.forTimeZone(c.getTimeZone()));
+ val = literal.getValue();
type = HiveParser.TOK_DATELITERAL;
DateFormat df = new SimpleDateFormat("yyyy-MM-dd");
- val = df.format(dt.toDateTime(DateTimeZone.getDefault()).toDate());
+ val = df.format(((Calendar) val).getTime());
val = "'" + val + "'";
}
break;
case TIME:
case TIMESTAMP: {
- //Calcite Calendar is always GMT, Hive atm uses JVM local
- final Calendar c = (Calendar) literal.getValue();
- final DateTime dt = new DateTime(c.getTimeInMillis(), DateTimeZone.forTimeZone(c.getTimeZone()));
+ val = literal.getValue();
type = HiveParser.TOK_TIMESTAMPLITERAL;
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
- val = df.format(dt.toDateTime(DateTimeZone.getDefault()).toDate());
+ val = df.format(((Calendar) val).getTime());
val = "'" + val + "'";
}
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
index 165f8c4..27990a2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.calcite.adapter.druid.DruidQuery;
+import org.apache.calcite.avatica.util.TimeUnitRange;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
index b1efbbd..e840938 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
import java.math.BigDecimal;
+import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
@@ -74,8 +75,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -292,17 +291,16 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
case DOUBLE:
return new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo,
Double.valueOf(((Number) literal.getValue3()).doubleValue()));
- case DATE: {
- final Calendar c = (Calendar) literal.getValue();
+ case DATE:
return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo,
- new java.sql.Date(c.getTimeInMillis()));
- }
+ new Date(((Calendar)literal.getValue()).getTimeInMillis()));
case TIME:
case TIMESTAMP: {
- final Calendar c = (Calendar) literal.getValue();
- final DateTime dt = new DateTime(c.getTimeInMillis(), DateTimeZone.forTimeZone(c.getTimeZone()));
- return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo,
- new Timestamp(dt.getMillis()));
+ Object value = literal.getValue3();
+ if (value instanceof Long) {
+ value = new Timestamp((Long)value);
+ }
+ return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, value);
}
case BINARY:
return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, literal.getValue3());
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index b9b600d..73a9b0f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
-import org.apache.hadoop.hive.ql.parse.*;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -74,8 +72,19 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
+import org.apache.hadoop.hive.ql.parse.JoinCond;
+import org.apache.hadoop.hive.ql.parse.JoinType;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
+import org.apache.hadoop.hive.ql.parse.PTFTranslator;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.UnparseTranslator;
+import org.apache.hadoop.hive.ql.parse.WindowingComponentizer;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -339,7 +348,6 @@ public class HiveOpConverter {
// through Hive
String[] baseSrc = new String[joinRel.getInputs().size()];
String tabAlias = getHiveDerivedTableAlias();
-
// 1. Convert inputs
OpAttr[] inputs = new OpAttr[joinRel.getInputs().size()];
List<Operator<?>> children = new ArrayList<Operator<?>>(joinRel.getInputs().size());
@@ -718,7 +726,7 @@ public class HiveOpConverter {
List<String> keepColNames) throws SemanticException {
// 1. Generate RS operator
// 1.1 Prune the tableNames, only count the tableNames that are not empty strings
- // as empty string in table aliases is only allowed for virtual columns.
+ // as empty string in table aliases is only allowed for virtual columns.
String tableAlias = null;
Set<String> tableNames = input.getSchema().getTableNames();
for (String tableName : tableNames) {
@@ -877,8 +885,7 @@ public class HiveOpConverter {
private static JoinOperator genJoin(RelNode join, ExprNodeDesc[][] joinExpressions,
List<List<ExprNodeDesc>> filterExpressions, List<Operator<?>> children,
- String[] baseSrc, String tabAlias)
- throws SemanticException {
+ String[] baseSrc, String tabAlias) throws SemanticException {
// 1. Extract join type
JoinCondDesc[] joinCondns;
@@ -1003,7 +1010,7 @@ public class HiveOpConverter {
// 4. We create the join operator with its descriptor
JoinDesc desc = new JoinDesc(exprMap, outputColumnNames, noOuterJoin, joinCondns,
- filters, joinExpressions, 0);
+ filters, joinExpressions);
desc.setReversedExprs(reversedExprs);
desc.setFilterMap(filterMap);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
index 52ca3b0..a05b89c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
@@ -23,12 +23,11 @@ import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
+import java.util.GregorianCalendar;
import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
-import org.apache.calcite.avatica.util.DateTimeUtils;
import org.apache.calcite.avatica.util.TimeUnit;
import org.apache.calcite.avatica.util.TimeUnitRange;
import org.apache.calcite.plan.RelOptCluster;
@@ -39,8 +38,8 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.sql.SqlCollation;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlKind;
@@ -77,10 +76,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeSubQueryDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
@@ -99,8 +96,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
@@ -253,8 +248,6 @@ public class RexNodeConverter {
boolean isWhenCase = tgtUdf instanceof GenericUDFWhen || tgtUdf instanceof GenericUDFCase;
boolean isTransformableTimeStamp = func.getGenericUDF() instanceof GenericUDFUnixTimeStamp &&
func.getChildren().size() != 0;
- boolean isBetween = !isNumeric && tgtUdf instanceof GenericUDFBetween;
- boolean isIN = !isNumeric && tgtUdf instanceof GenericUDFIn;
if (isNumeric) {
tgtDT = func.getTypeInfo();
@@ -273,33 +266,15 @@ public class RexNodeConverter {
} else if (isTransformableTimeStamp) {
// unix_timestamp(args) -> to_unix_timestamp(args)
func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), func.getChildren());
- } else if (isBetween) {
- assert func.getChildren().size() == 4;
- // We skip first child as is not involved (is the revert boolean)
- // The target type needs to account for all 3 operands
- tgtDT = FunctionRegistry.getCommonClassForComparison(
- func.getChildren().get(1).getTypeInfo(),
- FunctionRegistry.getCommonClassForComparison(
- func.getChildren().get(2).getTypeInfo(),
- func.getChildren().get(3).getTypeInfo()));
- } else if (isIN) {
- // We're only considering the first element of the IN list for the type
- assert func.getChildren().size() > 1;
- tgtDT = FunctionRegistry.getCommonClassForComparison(func.getChildren().get(0)
- .getTypeInfo(), func.getChildren().get(1).getTypeInfo());
}
- for (int i =0; i < func.getChildren().size(); ++i) {
- ExprNodeDesc childExpr = func.getChildren().get(i);
+ for (ExprNodeDesc childExpr : func.getChildren()) {
tmpExprNode = childExpr;
if (tgtDT != null
&& TypeInfoUtils.isConversionRequiredForComparison(tgtDT, childExpr.getTypeInfo())) {
- if (isCompare || isBetween || isIN) {
+ if (isCompare) {
// For compare, we will convert requisite children
- // For BETWEEN skip the first child (the revert boolean)
- if (!isBetween || i > 0) {
- tmpExprNode = ParseUtils.createConversionCast(childExpr, (PrimitiveTypeInfo) tgtDT);
- }
+ tmpExprNode = ParseUtils.createConversionCast(childExpr, (PrimitiveTypeInfo) tgtDT);
} else if (isNumeric) {
// For numeric, we'll do minimum necessary cast - if we cast to the type
// of expression, bad things will happen.
@@ -659,40 +634,20 @@ public class RexNodeConverter {
calciteLiteral = rexBuilder.makeCharLiteral(asUnicodeString((String) value));
break;
case DATE:
- // The Calcite literal is in GMT, this will be converted back to JVM locale
- // by ASTBuilder.literal during Calcite->Hive plan conversion
- final Calendar cal = Calendar.getInstance(DateTimeUtils.GMT_ZONE, Locale.getDefault());
- cal.setTime((Date) value);
- calciteLiteral = rexBuilder.makeDateLiteral(cal);
- break;
- case TIMESTAMP:
- // The Calcite literal is in GMT, this will be converted back to JVM locale
- // by ASTBuilder.literal during Calcite->Hive plan conversion
- final Calendar calt = Calendar.getInstance(DateTimeUtils.GMT_ZONE, Locale.getDefault());
- if (value instanceof Calendar) {
- final Calendar c = (Calendar) value;
- long timeMs = c.getTimeInMillis();
- calt.setTimeInMillis(timeMs);
- } else {
- final Timestamp ts = (Timestamp) value;
- // CALCITE-1690
- // Calcite cannot represent TIMESTAMP literals with precision higher than 3
- if (ts.getNanos() % 1000000 != 0) {
- throw new CalciteSemanticException(
- "High Precision Timestamp: " + String.valueOf(ts),
- UnsupportedFeature.HighPrecissionTimestamp);
- }
- calt.setTimeInMillis(ts.getTime());
- }
- // Must call makeLiteral, not makeTimestampLiteral
- // to have the RexBuilder.roundTime logic kick in
- calciteLiteral = rexBuilder.makeLiteral(
- calt,
- rexBuilder.getTypeFactory().createSqlType(
- SqlTypeName.TIMESTAMP,
- rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
- false);
- break;
+ Calendar cal = new GregorianCalendar();
+ cal.setTime((Date) value);
+ calciteLiteral = rexBuilder.makeDateLiteral(cal);
+ break;
+ case TIMESTAMP:
+ Calendar c = null;
+ if (value instanceof Calendar) {
+ c = (Calendar)value;
+ } else {
+ c = Calendar.getInstance();
+ c.setTimeInMillis(((Timestamp)value).getTime());
+ }
+ calciteLiteral = rexBuilder.makeTimestampLiteral(c, RelDataType.PRECISION_NOT_SPECIFIED);
+ break;
case INTERVAL_YEAR_MONTH:
// Calcite year-month literal value is months as BigDecimal
BigDecimal totalMonths = BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths());
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
index 10f5eb3..85450c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
@@ -352,7 +352,6 @@ public class SqlFunctionConverter {
registerFunction("struct", SqlStdOperatorTable.ROW, hToken(HiveParser.Identifier, "struct"));
registerFunction("isnotnull", SqlStdOperatorTable.IS_NOT_NULL, hToken(HiveParser.TOK_ISNOTNULL, "TOK_ISNOTNULL"));
registerFunction("isnull", SqlStdOperatorTable.IS_NULL, hToken(HiveParser.TOK_ISNULL, "TOK_ISNULL"));
- registerFunction("is not distinct from", SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, hToken(HiveParser.EQUAL_NS, "<=>"));
registerFunction("when", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when"));
registerDuplicateFunction("case", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when"));
// timebased
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
index 701bde4..2b075be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
@@ -194,7 +194,7 @@ public class ReduceSinkDeDuplication extends Transform {
ReduceSinkDesc cRSc = cRS.getConf();
for (ReduceSinkOperator pRSNs : pRSs) {
ReduceSinkDesc pRSNc = pRSNs.getConf();
- if (cRSc.getKeyCols().size() != pRSNc.getKeyCols().size()) {
+ if (cRSc.getKeyCols().size() < pRSNc.getKeyCols().size()) {
return false;
}
if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPrunerUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPrunerUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPrunerUtils.java
index 88b8119..4d3e74e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPrunerUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPrunerUtils.java
@@ -37,10 +37,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
public final class ListBucketingPrunerUtils {
/* Default list bucketing directory name. internal use only not for client. */
- public static final String HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME =
+ public static String HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME =
"HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME";
/* Default list bucketing directory key. internal use only not for client. */
- public static final String HIVE_LIST_BUCKETING_DEFAULT_KEY = "HIVE_DEFAULT_LIST_BUCKETING_KEY";
+ public static String HIVE_LIST_BUCKETING_DEFAULT_KEY = "HIVE_DEFAULT_LIST_BUCKETING_KEY";
/**
* Decide if pruner skips the skewed directory
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
index 2143718..6d0ee92 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
@@ -281,8 +281,7 @@ public final class GenMRSkewJoinProcessor {
MapJoinDesc mapJoinDescriptor = new MapJoinDesc(newJoinKeys, keyTblDesc,
newJoinValues, newJoinValueTblDesc, newJoinValueTblDesc,joinDescriptor
.getOutputColumnNames(), i, joinDescriptor.getConds(),
- joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin(), dumpFilePrefix,
- joinDescriptor.getNoConditionalTaskSize());
+ joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin(), dumpFilePrefix);
mapJoinDescriptor.setTagOrder(tags);
mapJoinDescriptor.setHandleSkewJoin(false);
mapJoinDescriptor.setNullSafes(joinDescriptor.getNullSafes());
@@ -384,11 +383,11 @@ public final class GenMRSkewJoinProcessor {
return true;
}
- private static final String skewJoinPrefix = "hive_skew_join";
- private static final String UNDERLINE = "_";
- private static final String BIGKEYS = "bigkeys";
- private static final String SMALLKEYS = "smallkeys";
- private static final String RESULTS = "results";
+ private static String skewJoinPrefix = "hive_skew_join";
+ private static String UNDERLINE = "_";
+ private static String BIGKEYS = "bigkeys";
+ private static String SMALLKEYS = "smallkeys";
+ private static String RESULTS = "results";
static Path getBigKeysDir(Path baseDir, Byte srcTbl) {
return StringInternUtils.internUriStringsInPath(
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
index b705f5a..38bb847 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
@@ -240,8 +240,7 @@ public class GenSparkSkewJoinProcessor {
MapJoinDesc mapJoinDescriptor = new MapJoinDesc(newJoinKeys, keyTblDesc,
newJoinValues, newJoinValueTblDesc, newJoinValueTblDesc, joinDescriptor
.getOutputColumnNames(), i, joinDescriptor.getConds(),
- joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin(), dumpFilePrefix,
- joinDescriptor.getNoConditionalTaskSize());
+ joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin(), dumpFilePrefix);
mapJoinDescriptor.setTagOrder(tags);
mapJoinDescriptor.setHandleSkewJoin(false);
mapJoinDescriptor.setNullSafes(joinDescriptor.getNullSafes());
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
deleted file mode 100644
index a5ed308..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer.physical;
-
-import java.util.concurrent.ExecutionException;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.registry.ServiceInstance;
-import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
-import org.apache.hadoop.hive.llap.registry.impl.InactiveServiceInstance;
-import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapClusterStateForCompile {
- protected static final Logger LOG = LoggerFactory.getLogger(LlapClusterStateForCompile.class);
-
- private static final long CLUSTER_UPDATE_INTERVAL_NS = 120 * 1000000000L; // 2 minutes.
- private Long lastClusterUpdateNs;
- private Integer noConfigNodeCount, executorCount;
- private int numExecutorsPerNode = -1;
- private LlapRegistryService svc;
- private final Configuration conf;
-
- // It's difficult to impossible to pass global things to compilation, so we have a static cache.
- private static final Cache<String, LlapClusterStateForCompile> CACHE =
- CacheBuilder.newBuilder().initialCapacity(10).maximumSize(100).build();
-
- public static LlapClusterStateForCompile getClusterInfo(final Configuration conf) {
- final String nodes = HiveConf.getTrimmedVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
- final String userName = HiveConf.getVar(
- conf, ConfVars.LLAP_ZK_REGISTRY_USER, LlapRegistryService.currentUser());
- Callable<LlapClusterStateForCompile> generator = new Callable<LlapClusterStateForCompile>() {
- @Override
- public LlapClusterStateForCompile call() throws Exception {
- LOG.info("Creating cluster info for " + userName + ":" + nodes);
- return new LlapClusterStateForCompile(conf);
- }
- };
- try {
- return CACHE.get(userName + ":" + nodes, generator);
- } catch (ExecutionException e) {
- throw new RuntimeException(e); // Should never happen... ctor is just assignments.
- }
- }
-
- private LlapClusterStateForCompile(Configuration conf) {
- this.conf = conf;
- }
-
- public boolean hasClusterInfo() {
- return lastClusterUpdateNs != null;
- }
-
- public int getKnownExecutorCount() {
- return executorCount;
- }
-
- public int getNodeCountWithUnknownExecutors() {
- return noConfigNodeCount;
- }
-
- public int getNumExecutorsPerNode() {
- return numExecutorsPerNode;
- }
-
- public synchronized void initClusterInfo() {
- if (lastClusterUpdateNs != null) {
- long elapsed = System.nanoTime() - lastClusterUpdateNs;
- if (elapsed < CLUSTER_UPDATE_INTERVAL_NS) return;
- }
- if (svc == null) {
- try {
- svc = LlapRegistryService.getClient(conf);
- } catch (Throwable t) {
- LOG.info("Cannot create the client; ignoring", t);
- return; // Don't fail; this is best-effort.
- }
- }
- ServiceInstanceSet instances;
- try {
- instances = svc.getInstances(10);
- } catch (IOException e) {
- LOG.info("Cannot update cluster information; ignoring", e);
- return; // Don't wait for the cluster if not started; this is best-effort.
- }
- int executorsLocal = 0, noConfigNodesLocal = 0;
- for (ServiceInstance si : instances.getAll()) {
- if (si instanceof InactiveServiceInstance) continue; // Shouldn't happen in getAll.
- Map<String, String> props = si.getProperties();
- if (props == null) {
- ++noConfigNodesLocal;
- continue;
- }
- try {
- int numExecutors = Integer.parseInt(props.get(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname));
- executorsLocal += numExecutors;
- if (numExecutorsPerNode == -1) {
- numExecutorsPerNode = numExecutors;
- }
- } catch (NumberFormatException e) {
- ++noConfigNodesLocal;
- }
- }
- lastClusterUpdateNs = System.nanoTime();
- noConfigNodeCount = noConfigNodesLocal;
- executorCount = executorsLocal;
- }
-}