You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/11/03 16:30:50 UTC
[4/5] hive git commit: HIVE-20617: Fix type of constants in IN
expressions to have correct type (Zoltan Haindrich via Jesus Camacho
Rodriguez)
HIVE-20617: Fix type of constants in IN expressions to have correct type (Zoltan Haindrich via Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83d8d4f1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83d8d4f1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83d8d4f1
Branch: refs/heads/master
Commit: 83d8d4f159edee28497184047369159e2ef28b8b
Parents: f3fab45
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Sat Nov 3 09:28:41 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Sat Nov 3 09:28:41 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/common/type/HiveChar.java | 7 +
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 15 +-
.../rules/HivePointLookupOptimizerRule.java | 249 +++++++++++-------
.../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 5 +-
.../hadoop/hive/ql/parse/TypeCheckCtx.java | 7 +-
.../hive/ql/parse/TypeCheckProcFactory.java | 247 +++++++++++++-----
.../ql/parse/TypeCheckProcFactoryUtils.java | 144 ++++++++++
.../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 35 ++-
ql/src/test/queries/clientnegative/udf_in_2.q | 2 -
.../groupby_multi_single_reducer3.q | 2 +
.../queries/clientpositive/in_typecheck_char.q | 24 ++
.../queries/clientpositive/in_typecheck_mixed.q | 12 +
.../clientpositive/in_typecheck_pointlook.q | 35 +++
.../clientpositive/in_typecheck_varchar.q | 16 ++
ql/src/test/queries/clientpositive/pcs.q | 2 -
.../queries/clientpositive/vector_struct_in.q | 2 +-
.../test/results/clientnegative/udf_in_2.q.out | 1 -
.../alter_partition_coltype.q.out | 2 +-
.../groupby_multi_single_reducer3.q.out | 16 +-
.../clientpositive/in_typecheck_char.q.out | 261 +++++++++++++++++++
.../clientpositive/in_typecheck_mixed.q.out | 99 +++++++
.../clientpositive/in_typecheck_pointlook.q.out | 181 +++++++++++++
.../clientpositive/in_typecheck_varchar.q.out | 185 +++++++++++++
.../clientpositive/infer_const_type.q.out | 8 +-
ql/src/test/results/clientpositive/join45.q.out | 4 +-
.../test/results/clientpositive/join45X.q.out | 256 ++++++++++++++++++
ql/src/test/results/clientpositive/join47.q.out | 4 +-
.../clientpositive/llap/acid_no_buckets.q.out | 32 +--
.../clientpositive/llap/bucketpruning1.q.out | 2 +-
.../clientpositive/llap/check_constraint.q.out | 6 +-
.../results/clientpositive/llap/dec_str.q.out | 10 +-
.../llap/dynpart_sort_optimization_acid.q.out | 49 ++--
.../clientpositive/llap/explainuser_1.q.out | 42 +--
.../clientpositive/llap/is_distinct_from.q.out | 2 +-
.../test/results/clientpositive/llap/kryo.q.out | 2 +-
.../results/clientpositive/llap/lineage3.q.out | 2 +-
.../clientpositive/llap/multi_column_in.q.out | 16 +-
.../llap/multi_column_in_single.q.out | 16 +-
.../clientpositive/llap/orc_llap_counters.q.out | 10 +-
.../llap/orc_predicate_pushdown.q.out | 26 +-
.../llap/parquet_predicate_pushdown.q.out | 26 +-
.../clientpositive/llap/vector_between_in.q.out | 18 +-
.../clientpositive/llap/vector_struct_in.q.out | 6 +-
.../clientpositive/llap/vectorization_13.q.out | 8 +-
.../clientpositive/llap/vectorization_6.q.out | 4 +-
.../clientpositive/llap/vectorization_8.q.out | 8 +-
.../llap/vectorization_short_regress.q.out | 8 +-
.../test/results/clientpositive/mapjoin47.q.out | 4 +-
.../parquet_vectorization_13.q.out | 8 +-
.../parquet_vectorization_6.q.out | 4 +-
.../parquet_vectorization_8.q.out | 8 +-
ql/src/test/results/clientpositive/pcs.q.out | 102 ++++----
.../clientpositive/perf/tez/cbo_query15.q.out | 2 +-
.../perf/tez/constraints/cbo_query15.q.out | 2 +-
.../results/clientpositive/ppd_udf_col.q.out | 10 +-
.../spark/groupby_multi_single_reducer3.q.out | 16 +-
.../spark/parquet_vectorization_13.q.out | 8 +-
.../spark/parquet_vectorization_6.q.out | 4 +-
.../spark/parquet_vectorization_8.q.out | 8 +-
.../spark/spark_explainuser_1.q.out | 42 +--
.../spark/vector_between_in.q.out | 18 +-
.../clientpositive/spark/vectorization_13.q.out | 8 +-
.../clientpositive/spark/vectorization_6.q.out | 4 +-
.../spark/vectorization_short_regress.q.out | 8 +-
.../vector_non_constant_in_expr.q.out | 4 +-
.../clientpositive/vector_struct_in.q.out | 6 +-
.../clientpositive/vectorization_13.q.out | 8 +-
.../clientpositive/vectorization_6.q.out | 4 +-
.../clientpositive/vectorization_8.q.out | 8 +-
70 files changed, 1926 insertions(+), 476 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
index 29dc06d..f0b28c7 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
@@ -43,6 +43,7 @@ public class HiveChar extends HiveBaseChar
/**
* Set char value, padding or truncating the value to the size of len parameter.
*/
+ @Override
public void setValue(String val, int len) {
super.setValue(HiveBaseChar.getPaddedValue(val, len), -1);
}
@@ -59,15 +60,18 @@ public class HiveChar extends HiveBaseChar
return value;
}
+ @Override
public int getCharacterLength() {
String strippedValue = getStrippedValue();
return strippedValue.codePointCount(0, strippedValue.length());
}
+ @Override
public String toString() {
return getPaddedValue();
}
+ @Override
public int compareTo(HiveChar rhs) {
if (rhs == this) {
return 0;
@@ -75,6 +79,7 @@ public class HiveChar extends HiveBaseChar
return this.getStrippedValue().compareTo(rhs.getStrippedValue());
}
+ @Override
public boolean equals(Object rhs) {
if (rhs == this) {
return true;
@@ -85,7 +90,9 @@ public class HiveChar extends HiveBaseChar
return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue());
}
+ @Override
public int hashCode() {
return getStrippedValue().hashCode();
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index e7d7159..1a58311 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -18,13 +18,6 @@
package org.apache.hadoop.hive.ql;
-import java.io.FileNotFoundException;
-import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
@@ -35,6 +28,13 @@ import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin;
import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.security.AccessControlException;
+import java.io.FileNotFoundException;
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/**
* List of all error messages.
* This list contains both compile time and run-time errors.
@@ -469,6 +469,7 @@ public enum ErrorMsg {
LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"),
RESOURCE_PLAN_ALREADY_EXISTS(10417, "Resource plan {0} already exists", true),
RESOURCE_PLAN_NOT_EXISTS(10418, "Resource plan {0} does not exist", true),
+ INCOMPATIBLE_STRUCT(10419, "Incompatible structs.", true),
//========================== 20000 range starts here ========================//
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
index 04800cc..9a525ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
@@ -20,25 +20,24 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.AbstractRelNode;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexUtil;
@@ -59,15 +58,18 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
-public abstract class HivePointLookupOptimizerRule extends RelOptRule {
-
/**
- * This optimization will take a Filter or expression, and if its predicate contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
+ * This optimization attempts to identify and close expanded INs.
+ *
+ * Basically:
+ * <pre>
+ * (c) IN ( v1, v2, ...) <=> c1=v1 || c1=v2 || ...
+ * </pre>
+ * If c is struct; then c=v1 is a group of anded equations.
*/
+public abstract class HivePointLookupOptimizerRule extends RelOptRule {
+
+ /** Rule adapter to apply the transformation to Filter conditions. */
public static class FilterCondition extends HivePointLookupOptimizerRule {
public FilterCondition (int minNumORClauses) {
super(operand(Filter.class, any()), minNumORClauses);
@@ -78,22 +80,20 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
final Filter filter = call.rel(0);
final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
- analyzeCondition(call , rexBuilder, filter, condition);
- }
- @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
- final Filter filter = (Filter) node;
- return filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
+ RexNode newCondition = analyzeRexNode(rexBuilder, condition);
+
+ // If we could not transform anything, we bail out
+ if (newCondition.toString().equals(condition.toString())) {
+ return;
+ }
+ RelNode newNode = filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
+
+ call.transformTo(newNode);
}
}
-/**
- * This optimization will take a Join or expression, and if its join condition contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
- */
+ /** Rule adapter to apply the transformation to Join conditions. */
public static class JoinCondition extends HivePointLookupOptimizerRule {
public JoinCondition (int minNumORClauses) {
super(operand(Join.class, any()), minNumORClauses);
@@ -104,18 +104,55 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
final Join join = call.rel(0);
final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
final RexNode condition = RexUtil.pullFactors(rexBuilder, join.getCondition());
- analyzeCondition(call , rexBuilder, join, condition);
+
+ RexNode newCondition = analyzeRexNode(rexBuilder, condition);
+
+ // If we could not transform anything, we bail out
+ if (newCondition.toString().equals(condition.toString())) {
+ return;
+ }
+
+ RelNode newNode = join.copy(join.getTraitSet(),
+ newCondition,
+ join.getLeft(),
+ join.getRight(),
+ join.getJoinType(),
+ join.isSemiJoinDone());
+
+ call.transformTo(newNode);
+ }
+ }
+
+ /** Rule adapter to apply the transformation to Projections. */
+ public static class ProjectionExpressions extends HivePointLookupOptimizerRule {
+ public ProjectionExpressions(int minNumORClauses) {
+ super(operand(Project.class, any()), minNumORClauses);
}
- @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
- final Join join = (Join) node;
- return join.copy(join.getTraitSet(),
- newCondition,
- join.getLeft(),
- join.getRight(),
- join.getJoinType(),
- join.isSemiJoinDone());
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ boolean changed = false;
+ final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
+ List<RexNode> newProjects = new ArrayList<>();
+ for (RexNode oldNode : project.getProjects()) {
+ RexNode newNode = analyzeRexNode(rexBuilder, oldNode);
+ if (!newNode.toString().equals(oldNode.toString())) {
+ changed = true;
+ newProjects.add(newNode);
+ } else {
+ newProjects.add(oldNode);
+ }
+ }
+ if (!changed) {
+ return;
+ }
+ Project newProject = project.copy(project.getTraitSet(), project.getInput(), newProjects,
+ project.getRowType(), project.getFlags());
+ call.transformTo(newProject);
+
}
+
}
protected static final Logger LOG = LoggerFactory.getLogger(HivePointLookupOptimizerRule.class);
@@ -123,37 +160,21 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
// Minimum number of OR clauses needed to transform into IN clauses
protected final int minNumORClauses;
- protected abstract RelNode copyNode(AbstractRelNode node, RexNode newCondition);
-
protected HivePointLookupOptimizerRule(
RelOptRuleOperand operand, int minNumORClauses) {
super(operand);
this.minNumORClauses = minNumORClauses;
}
- public void analyzeCondition(RelOptRuleCall call,
- RexBuilder rexBuilder,
- AbstractRelNode node,
- RexNode condition) {
-
+ public RexNode analyzeRexNode(RexBuilder rexBuilder, RexNode condition) {
// 1. We try to transform possible candidates
- RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, node,
- minNumORClauses);
+ RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, minNumORClauses);
RexNode newCondition = transformIntoInClause.apply(condition);
// 2. We merge IN expressions
RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder);
newCondition = mergeInClause.apply(newCondition);
-
- // 3. If we could not transform anything, we bail out
- if (newCondition.toString().equals(condition.toString())) {
- return;
- }
-
- // 4. We create the Filter/Join with the new condition
- RelNode newNode = copyNode(node, newCondition);
-
- call.transformTo(newNode);
+ return newCondition;
}
@@ -162,11 +183,9 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
*/
protected static class RexTransformIntoInClause extends RexShuttle {
private final RexBuilder rexBuilder;
- private final AbstractRelNode nodeOp;
private final int minNumORClauses;
- RexTransformIntoInClause(RexBuilder rexBuilder, AbstractRelNode nodeOp, int minNumORClauses) {
- this.nodeOp = nodeOp;
+ RexTransformIntoInClause(RexBuilder rexBuilder, int minNumORClauses) {
this.rexBuilder = rexBuilder;
this.minNumORClauses = minNumORClauses;
}
@@ -180,7 +199,7 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
case OR:
try {
RexNode newNode = transformIntoInClauseCondition(rexBuilder,
- nodeOp.getRowType(), call, minNumORClauses);
+ call, minNumORClauses);
if (newNode != null) {
return newNode;
}
@@ -196,18 +215,56 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
}
/**
- * Represents a simple contraint.
+ * This class just wraps around a RexNode enables equals/hashCode based on toString.
+ *
+ * After CALCITE-2632 this might not be needed anymore */
+ static class RexNodeRef {
+
+ public static Comparator<RexNodeRef> COMPARATOR =
+ (RexNodeRef o1, RexNodeRef o2) -> o1.node.toString().compareTo(o2.node.toString());
+ private RexNode node;
+
+ public RexNodeRef(RexNode node) {
+ this.node = node;
+ }
+
+ public RexNode getRexNode() {
+ return node;
+ }
+
+ @Override
+ public int hashCode() {
+ return node.toString().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof RexNodeRef) {
+ RexNodeRef otherRef = (RexNodeRef) o;
+ return node.toString().equals(otherRef.node.toString());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "ref for:" + node.toString();
+ }
+ }
+ /**
+ * Represents a contraint.
*
* Example: a=1
+ * substr(a,1,2) = concat('asd','xxx')
*/
static class Constraint {
- private RexLiteral literal;
- private RexInputRef inputRef;
+ private RexNode exprNode;
+ private RexNode constNode;
- public Constraint(RexInputRef inputRef, RexLiteral literal) {
- this.literal = literal;
- this.inputRef = inputRef;
+ public Constraint(RexNode exprNode, RexNode constNode) {
+ this.exprNode = constNode;
+ this.constNode = exprNode;
}
/**
@@ -223,21 +280,31 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
}
RexNode opA = call.operands.get(0);
RexNode opB = call.operands.get(1);
- if (opA instanceof RexLiteral && opB instanceof RexInputRef) {
- RexLiteral rexLiteral = (RexLiteral) opA;
- RexInputRef rexInputRef = (RexInputRef) opB;
- return new Constraint(rexInputRef, rexLiteral);
+ if (RexUtil.isNull(opA) || RexUtil.isNull(opB)) {
+ // dont try to compare nulls
+ return null;
+ }
+ if (isConstExpr(opA) && isColumnExpr(opB)) {
+ return new Constraint(opB, opA);
}
- if (opA instanceof RexInputRef && opB instanceof RexLiteral) {
- RexLiteral rexLiteral = (RexLiteral) opB;
- RexInputRef rexInputRef = (RexInputRef) opA;
- return new Constraint(rexInputRef, rexLiteral);
+ if (isColumnExpr(opA) && isConstExpr(opB)) {
+ return new Constraint(opA, opB);
}
return null;
}
- public RexInputRef getKey() {
- return inputRef;
+ private static boolean isColumnExpr(RexNode node) {
+ return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() > 0
+ && HiveCalciteUtil.isDeterministic(node);
+ }
+
+ private static boolean isConstExpr(RexNode node) {
+ return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() == 0
+ && HiveCalciteUtil.isDeterministic(node);
+ }
+
+ public RexNodeRef getKey() {
+ return new RexNodeRef(constNode);
}
}
@@ -254,17 +321,17 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
*
*/
static class ConstraintGroup {
+ public static final Function<ConstraintGroup, Set<RexNodeRef>> KEY_FUNCTION =
+ new Function<ConstraintGroup, Set<RexNodeRef>>() {
- public static final Function<ConstraintGroup, Set<RexInputRef>> KEY_FUNCTION = new Function<ConstraintGroup, Set<RexInputRef>>() {
-
- @Override
- public Set<RexInputRef> apply(ConstraintGroup a) {
- return a.key;
- }
- };
- private Map<RexInputRef, Constraint> constraints = new HashMap<>();
+ @Override
+ public Set<RexNodeRef> apply(ConstraintGroup cg) {
+ return cg.key;
+ }
+ };
+ private Map<RexNodeRef, Constraint> constraints = new HashMap<>();
private RexNode originalRexNode;
- private final Set<RexInputRef> key;
+ private final Set<RexNodeRef> key;
public ConstraintGroup(RexNode rexNode) {
originalRexNode = rexNode;
@@ -289,21 +356,21 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
key = constraints.keySet();
}
- public List<RexNode> getValuesInOrder(List<RexInputRef> columns) throws SemanticException {
+ public List<RexNode> getValuesInOrder(List<RexNodeRef> columns) throws SemanticException {
List<RexNode> ret = new ArrayList<>();
- for (RexInputRef rexInputRef : columns) {
+ for (RexNodeRef rexInputRef : columns) {
Constraint constraint = constraints.get(rexInputRef);
if (constraint == null) {
throw new SemanticException("Unable to find constraint which was earlier added.");
}
- ret.add(constraint.literal);
+ ret.add(constraint.exprNode);
}
return ret;
}
}
- private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataType inputSchema,
- RexNode condition, int minNumORClauses) throws SemanticException {
+ private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RexNode condition,
+ int minNumORClauses) throws SemanticException {
assert condition.getKind() == SqlKind.OR;
ImmutableList<RexNode> operands = RexUtil.flattenOr(((RexCall) condition).getOperands());
@@ -318,10 +385,10 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
allNodes.add(m);
}
- Multimap<Set<RexInputRef>, ConstraintGroup> assignmentGroups =
+ Multimap<Set<RexNodeRef>, ConstraintGroup> assignmentGroups =
Multimaps.index(allNodes, ConstraintGroup.KEY_FUNCTION);
- for (Entry<Set<RexInputRef>, Collection<ConstraintGroup>> sa : assignmentGroups.asMap().entrySet()) {
+ for (Entry<Set<RexNodeRef>, Collection<ConstraintGroup>> sa : assignmentGroups.asMap().entrySet()) {
// skip opaque
if (sa.getKey().size() == 0) {
continue;
@@ -351,13 +418,15 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
}
- private RexNode buildInFor(Set<RexInputRef> set, Collection<ConstraintGroup> value) throws SemanticException {
+ private RexNode buildInFor(Set<RexNodeRef> set, Collection<ConstraintGroup> value) throws SemanticException {
- List<RexInputRef> columns = new ArrayList<RexInputRef>();
+ List<RexNodeRef> columns = new ArrayList<>();
columns.addAll(set);
+ columns.sort(RexNodeRef.COMPARATOR);
List<RexNode >operands = new ArrayList<>();
- operands.add(useStructIfNeeded(columns));
+ List<RexNode> columnNodes = columns.stream().map(n -> n.getRexNode()).collect(Collectors.toList());
+ operands.add(useStructIfNeeded(columnNodes));
for (ConstraintGroup node : value) {
List<RexNode> values = node.getValuesInOrder(columns);
operands.add(useStructIfNeeded(values));
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index c9df668..3a51d97 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -910,7 +910,7 @@ public abstract class BaseSemanticAnalyzer {
} catch(Exception e) {
throw new SemanticException(
ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("Invalid CHECK constraint expression: ")
- + cc.getCheck_expression() + ". " + e.getMessage());
+ + cc.getCheck_expression() + ". " + e.getMessage(), e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 82e975a..d92cfca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+
import org.antlr.runtime.ClassicToken;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
@@ -277,7 +278,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.joda.time.Interval;
-import javax.sql.DataSource;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@@ -304,6 +304,8 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import javax.sql.DataSource;
+
public class CalcitePlanner extends SemanticAnalyzer {
@@ -2064,6 +2066,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
if (conf.getBoolVar(HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {
rules.add(new HivePointLookupOptimizerRule.FilterCondition(minNumORClauses));
rules.add(new HivePointLookupOptimizerRule.JoinCondition(minNumORClauses));
+ rules.add(new HivePointLookupOptimizerRule.ProjectionExpressions(minNumORClauses));
}
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_CONSTRAINTS_JOIN) &&
profilesCBO.contains(ExtendedCBOProfile.REFERENTIAL_CONSTRAINTS)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
index 1b56ecd..12e7ae6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
@@ -19,9 +19,10 @@
package org.apache.hadoop.hive.ql.parse;
import org.apache.calcite.rel.RelNode;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
import java.util.Map;
/**
@@ -263,4 +264,8 @@ public class TypeCheckCtx implements NodeProcessorCtx {
public boolean isFoldExpr() {
return foldExpr;
}
+
+ public boolean isCBOExecuted() {
+ return foldExpr;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index 4968d16..fd151b9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
@@ -353,14 +354,18 @@ public class TypeCheckProcFactory {
}
public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) {
- // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
- // precision and scale of the value by throwing away trailing zeroes. This may or may
- // not be desirable for the literals; however, this used to be the default behavior
- // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
HiveDecimal hd = HiveDecimal.create(strVal);
if (notNull && hd == null) {
return null;
}
+ return new ExprNodeConstantDesc(adjustType(hd), hd);
+ }
+
+ private static DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
int prec = 1;
int scale = 0;
if (hd != null) {
@@ -368,7 +373,7 @@ public class TypeCheckProcFactory {
scale = hd.scale();
}
DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
- return new ExprNodeConstantDesc(typeInfo, hd);
+ return typeInfo;
}
}
@@ -1163,36 +1168,59 @@ public class TypeCheckProcFactory {
ExprNodeDesc constChild = children.get(constIdx);
ExprNodeDesc columnChild = children.get(1 - constIdx);
- final PrimitiveTypeInfo colTypeInfo =
- TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase());
- ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild);
- if (newChild == null) {
- // non-interpretabe as that type...
- if (genericUDF instanceof GenericUDFOPEqual) {
- return new ExprNodeConstantDesc(false);
- }
- } else {
- children.set(constIdx, newChild);
+ final PrimitiveTypeInfo colTypeInfo =
+ TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase());
+ ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild);
+ if (newChild == null) {
+ // non-interpretable as target type...
+ // TODO: all comparisons with null should result in null
+ if (genericUDF instanceof GenericUDFOPEqual
+ && !(genericUDF instanceof GenericUDFOPEqualNS)) {
+ return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null);
}
+ } else {
+ children.set(constIdx, newChild);
+ }
}
- if (genericUDF instanceof GenericUDFIn && children.get(0) instanceof ExprNodeColumnDesc) {
- ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) children.get(0);
- final PrimitiveTypeInfo colTypeInfo =
- TypeInfoFactory.getPrimitiveTypeInfo(columnDesc.getTypeString().toLowerCase());
+ if (genericUDF instanceof GenericUDFIn) {
+
+ ExprNodeDesc columnDesc = children.get(0);
List<ExprNodeDesc> outputOpList = children.subList(1, children.size());
ArrayList<ExprNodeDesc> inOperands = new ArrayList<>(outputOpList);
outputOpList.clear();
+ boolean hasNullValue = false;
for (ExprNodeDesc oldChild : inOperands) {
- if(oldChild !=null && oldChild instanceof ExprNodeConstantDesc) {
- ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, oldChild);
- if(newChild == null) {
- // non interpretable as target type; skip
- continue;
+ if (oldChild == null) {
+ hasNullValue = true;
+ continue;
+ }
+ ExprNodeDesc newChild = interpretNodeAsStruct(columnDesc, oldChild);
+ if (newChild == null) {
+ hasNullValue = true;
+ continue;
+ }
+ outputOpList.add(newChild);
+ }
+
+ if (hasNullValue) {
+ ExprNodeConstantDesc nullConst = new ExprNodeConstantDesc(columnDesc.getTypeInfo(), null);
+ if (outputOpList.size() == 0) {
+ // we have found only null values...remove the IN ; it will be null all the time.
+ return nullConst;
+ }
+ outputOpList.add(nullConst);
+ }
+ if (!ctx.isCBOExecuted()) {
+ ArrayList<ExprNodeDesc> orOperands = TypeCheckProcFactoryUtils.rewriteInToOR(children);
+ if (orOperands != null) {
+ if (orOperands.size() == 1) {
+ orOperands.add(new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false));
}
- outputOpList.add(newChild);
- }else{
- outputOpList.add(oldChild);
+ funcText = "or";
+ genericUDF = new GenericUDFOPOr();
+ children.clear();
+ children.addAll(orOperands);
}
}
}
@@ -1258,48 +1286,145 @@ public class TypeCheckProcFactory {
return desc;
}
- private ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) {
+ /**
+ * Interprets the given value as columnDesc if possible
+ */
+ private static ExprNodeDesc interpretNodeAsStruct(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc)
+ throws SemanticException {
+ if(columnDesc instanceof ExprNodeColumnDesc) {
+ ExprNodeColumnDesc exprNodeColumnDesc = (ExprNodeColumnDesc) columnDesc;
+ final PrimitiveTypeInfo typeInfo =
+ TypeInfoFactory.getPrimitiveTypeInfo(exprNodeColumnDesc.getTypeString().toLowerCase());
+ return interpretNodeAs(typeInfo, valueDesc);
+ }
+ if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isConstantStruct(valueDesc)) {
+ List<ExprNodeDesc> columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren();
+ ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc;
+ StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo();
+ ArrayList<TypeInfo> structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos();
+ ArrayList<TypeInfo> newStructFieldInfos = new ArrayList<>();
+
+ if (columnChilds.size() != structFieldInfos.size()) {
+ throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + structFieldInfos));
+ }
+ List<Object> oldValues = (List<Object>) valueConstDesc.getValue();
+ List<Object> newValues = new ArrayList<>();
+ for (int i = 0; i < columnChilds.size(); i++) {
+ newStructFieldInfos.add(columnChilds.get(i).getTypeInfo());
+ Object newValue = interpretConstantAsPrimitive(
+ (PrimitiveTypeInfo) columnChilds.get(i).getTypeInfo(),
+ oldValues.get(i),
+ structFieldInfos.get(i));
+ newValues.add(newValue);
+ }
+ StructTypeInfo sti = new StructTypeInfo();
+ sti.setAllStructFieldTypeInfos(newStructFieldInfos);
+ sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames());
+ return new ExprNodeConstantDesc(sti, newValues);
+
+ }
+ if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isStructUDF(valueDesc)) {
+ List<ExprNodeDesc> columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren();
+ List<ExprNodeDesc> valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren();
+ if (columnChilds.size() != valueChilds.size()) {
+ throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + valueChilds));
+ }
+ List<ExprNodeDesc> oldValueChilds = new ArrayList<>(valueChilds);
+ valueChilds.clear();
+ for (int i = 0; i < oldValueChilds.size(); i++) {
+ ExprNodeDesc newValue = interpretNodeAsStruct(columnChilds.get(i), oldValueChilds.get(i));
+ valueChilds.add(newValue);
+ }
+ }
+ return valueDesc;
+ }
+
+ private static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) {
if (constChild instanceof ExprNodeConstantDesc) {
// Try to narrow type of constant
Object constVal = ((ExprNodeConstantDesc) constChild).getValue();
- String constType = constChild.getTypeString().toLowerCase();
- if (constVal instanceof Number || constVal instanceof String) {
- try {
- PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry();
- if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
- return new ExprNodeConstantDesc(new Integer(constVal.toString()));
- } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
- return new ExprNodeConstantDesc(new Long(constVal.toString()));
- } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
- return new ExprNodeConstantDesc(new Double(constVal.toString()));
- } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
- return new ExprNodeConstantDesc(new Float(constVal.toString()));
- } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
- return new ExprNodeConstantDesc(new Byte(constVal.toString()));
- } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
- return new ExprNodeConstantDesc(new Short(constVal.toString()));
- } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
- return NumExprProcessor.createDecimal(constVal.toString(), false);
- }
- } catch (NumberFormatException nfe) {
- LOG.trace("Failed to narrow type of constant", nfe);
- if (!NumberUtils.isNumber(constVal.toString())) {
- return null;
- }
+ if (constVal == null) {
+ // adjust type of null
+ return new ExprNodeConstantDesc(colTypeInfo, null);
+ }
+ Object newConst = interpretConstantAsPrimitive(colTypeInfo, constVal, constChild.getTypeInfo());
+ if (newConst == null) {
+ return null;
+ }
+ if(newConst == constVal) {
+ return constChild;
+ } else {
+ return new ExprNodeConstantDesc(adjustType(colTypeInfo, newConst), newConst);
+ }
+ }
+ return constChild;
+ }
+
+ private static TypeInfo adjustType(PrimitiveTypeInfo colTypeInfo, Object newConst) {
+ if (newConst instanceof HiveDecimal) {
+ return NumExprProcessor.adjustType((HiveDecimal) newConst);
+ }
+ return colTypeInfo;
+ }
+
+ private static Object interpretConstantAsPrimitive(PrimitiveTypeInfo colTypeInfo, Object constVal,
+ TypeInfo constTypeInfo) {
+ String constTypeInfoName = constTypeInfo.getTypeName();
+ if (constVal instanceof Number || constVal instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return (new Integer(constVal.toString()));
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return (new Long(constVal.toString()));
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return (new Double(constVal.toString()));
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return (new Float(constVal.toString()));
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return (new Byte(constVal.toString()));
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return (new Short(constVal.toString()));
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ return HiveDecimal.create(constVal.toString());
+ }
+ } catch (NumberFormatException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ if (!NumberUtils.isNumber(constVal.toString())) {
+ return null;
}
}
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constVal instanceof HiveDecimal) {
+ HiveDecimal hiveDecimal = (HiveDecimal) constVal;
- // if column type is char and constant type is string, then convert the constant to char
- // type with padded spaces.
- if (constType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) {
- final Object originalValue = ((ExprNodeConstantDesc) constChild).getValue();
- final String constValue = originalValue.toString();
- final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo);
- final HiveChar newValue = new HiveChar(constValue, length);
- return new ExprNodeConstantDesc(colTypeInfo, newValue);
+ PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return hiveDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return hiveDecimal.floatValue();
}
+ return hiveDecimal;
}
- return constChild;
+
+ // TODO : Char and string comparison happens in char. But, varchar and string comparison happens in String.
+
+ // if column type is char and constant type is string, then convert the constant to char
+ // type with padded spaces.
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) {
+ final String constValue = constVal.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return newValue;
+ } else {
+ return null;
+ }
+ }
+ return constVal;
}
private boolean canConvertIntoNvl(GenericUDF genericUDF, ArrayList<ExprNodeDesc> children) {
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java
new file mode 100644
index 0000000..b0544f3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
+import com.google.common.collect.Lists;
+
+public class TypeCheckProcFactoryUtils {
+
+ static ArrayList<ExprNodeDesc> rewriteInToOR(ArrayList<ExprNodeDesc> inOperands) throws SemanticException {
+ ExprNodeDesc columnDesc = inOperands.get(0);
+
+ ArrayList<ExprNodeDesc> orOperands = new ArrayList<>();
+ for (int i = 1; i < inOperands.size(); i++) {
+ ExprNodeDesc andExpr = buildEqualsArr(columnDesc, inOperands.get(i));
+ if (andExpr == null) {
+ return null;
+ }
+ orOperands.add(andExpr);
+ }
+ return orOperands;
+ }
+
+ private static ExprNodeDesc buildEqualsArr(ExprNodeDesc columnDesc, ExprNodeDesc exprNodeDesc)
+ throws SemanticException {
+ List<ExprNodeDesc> lNodes = asListOfNodes(columnDesc);
+ List<ExprNodeDesc> rNodes = asListOfNodes(exprNodeDesc);
+ if (lNodes == null || rNodes == null) {
+ // something went wrong
+ return null;
+ }
+ if (lNodes.size() != rNodes.size()) {
+ throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnDesc + " and " + exprNodeDesc));
+ }
+
+ List<ExprNodeDesc> ret = new ArrayList<>();
+ for (int i = 0; i < lNodes.size(); i++) {
+ ret.add(buildEquals(lNodes.get(i), rNodes.get(i)));
+ }
+ return buildAnd(ret);
+ }
+
+ private static ExprNodeGenericFuncDesc buildEquals(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPEqual(), "=",
+ Lists.newArrayList(columnDesc, valueDesc));
+ }
+
+ private static ExprNodeDesc buildAnd(List<ExprNodeDesc> values) {
+ if (values.size() == 1) {
+ return values.get(0);
+ } else {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(), "and", values);
+ }
+ }
+
+ private static List<ExprNodeDesc> asListOfNodes(ExprNodeDesc desc) {
+ ExprNodeDesc valueDesc = desc;
+ if (ExprNodeDescUtils.isStructUDF(desc)) {
+ List<ExprNodeDesc> valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren();
+ for (ExprNodeDesc exprNodeDesc : valueChilds) {
+ if (!isSafeExpression(exprNodeDesc)) {
+ return null;
+ }
+ }
+ return valueChilds;
+ }
+ if (ExprNodeDescUtils.isConstantStruct(valueDesc)) {
+ ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc;
+ List<Object> oldValues = (List<Object>) valueConstDesc.getValue();
+ StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo();
+ ArrayList<TypeInfo> structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos();
+
+ List<ExprNodeDesc> ret = new ArrayList<>();
+ for (int i = 0; i < oldValues.size(); i++) {
+ ret.add(new ExprNodeConstantDesc(structFieldInfos.get(i), oldValues.get(i)));
+ }
+ return ret;
+ }
+ if (isSafeExpression(desc)) {
+ return Lists.newArrayList(desc);
+ }
+
+ return null;
+ }
+
+ private static boolean isSafeExpression(ExprNodeDesc desc) {
+ TypeInfo typeInfo = desc.getTypeInfo();
+ if (typeInfo.getCategory() != Category.PRIMITIVE) {
+ return false;
+ }
+ if (isConstantOrColumn(desc)) {
+ return true;
+ }
+ if (desc instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) desc;
+ if (FunctionRegistry.isConsistentWithinQuery(exprNodeGenericFuncDesc.getGenericUDF())) {
+ for (ExprNodeDesc child : exprNodeGenericFuncDesc.getChildren()) {
+ if (!isSafeExpression(child)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isConstantOrColumn(ExprNodeDesc desc) {
+ return desc instanceof ExprNodeColumnDesc || desc instanceof ExprNodeConstantDesc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
index c274fd7..596640e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
@@ -18,18 +18,14 @@
package org.apache.hadoop.hive.ql.plan;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -38,6 +34,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -45,10 +42,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.Pr
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.util.ReflectionUtils;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
public class ExprNodeDescUtils {
@@ -571,7 +573,7 @@ public class ExprNodeDescUtils {
/**
* Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc.
- *
+ *
* @param exprDesc
* @param hashCodeToColumnDescMap
* Assumption: If two ExprNodeColumnDesc have same hash code then
@@ -642,7 +644,9 @@ public class ExprNodeDescUtils {
// We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
// We also don't do anything for non-primitive children (maybe we should assert).
if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
- || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
+ || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) {
+ return pti;
+ }
PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
// If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
@@ -652,7 +656,7 @@ public class ExprNodeDescUtils {
* Build ExprNodeColumnDesc for the projections in the input operator from
* sartpos to endpos(both included). Operator must have an associated
* colExprMap.
- *
+ *
* @param inputOp
* Input Hive Operator
* @param startPos
@@ -684,7 +688,7 @@ public class ExprNodeDescUtils {
}
return exprColLst;
- }
+ }
public static List<ExprNodeDesc> flattenExprList(List<ExprNodeDesc> sourceList) {
ArrayList<ExprNodeDesc> result = new ArrayList<ExprNodeDesc>(sourceList.size());
@@ -998,4 +1002,17 @@ public class ExprNodeDescUtils {
}
return false;
}
+
+ public static boolean isConstantStruct(ExprNodeDesc valueDesc) {
+ return valueDesc instanceof ExprNodeConstantDesc && valueDesc.getTypeInfo() instanceof StructTypeInfo;
+ }
+
+ public static boolean isStructUDF(ExprNodeDesc columnDesc) {
+ if (columnDesc instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) columnDesc;
+ return (exprNodeGenericFuncDesc.getGenericUDF() instanceof GenericUDFStruct);
+ }
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientnegative/udf_in_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/udf_in_2.q b/ql/src/test/queries/clientnegative/udf_in_2.q
deleted file mode 100644
index 2288054..0000000
--- a/ql/src/test/queries/clientnegative/udf_in_2.q
+++ /dev/null
@@ -1,2 +0,0 @@
-select 1=1 in (true, false);
-
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q b/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
index 94aea35..03af1cc 100644
--- a/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
+++ b/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
@@ -3,6 +3,8 @@
-- SORT_QUERY_RESULTS
+set hive.cbo.enable=true;
+
create table e1_n1 (key string, count int);
create table e2_n2 (key string, count int);
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_char.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_char.q b/ql/src/test/queries/clientpositive/in_typecheck_char.q
new file mode 100644
index 0000000..3955c4b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_char.q
@@ -0,0 +1,24 @@
+
+create table ax(s char(1),t char(10));
+
+insert into ax values ('a','a'),('a','a '),('b','bb');
+
+explain
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a';
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a';
+
+explain
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+
+select 'expected 2',count(*) from ax where t = 'a ';
+select 'expected 2',count(*) from ax where t = 'a ';
+select 'expected 0',count(*) from ax where t = 'a d';
+
+
+select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb'));
+
+
+-- this is right now broken; HIVE-20779 should fix it
+explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null;
+select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null;
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_mixed.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_mixed.q b/ql/src/test/queries/clientpositive/in_typecheck_mixed.q
new file mode 100644
index 0000000..26c486b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_mixed.q
@@ -0,0 +1,12 @@
+create table t (a string);
+
+insert into t values ('1'),('x'),('2.0');
+
+explain
+select * from t where a in (1.0,'x',2);
+select * from t where a in (1.0,'x',2);
+
+
+create table ax(s char(1),t char(10));
+insert into ax values ('a','a'),('a','a '),('b','bb');
+select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null;
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q b/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q
new file mode 100644
index 0000000..c788506
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q
@@ -0,0 +1,35 @@
+
+create table customer_demographics (cd_marital_status char(1), cd_education_status char(20));
+
+insert into customer_demographics values
+('M','Unknown'),
+('W','Advanced Degree'),
+('W','Advanced Degree '),
+('W',' Advanced Degree')
+;
+
+set hive.optimize.point.lookup.min=32;
+
+explain
+select count(1)
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
+
+select '3 is expected:',count(1)
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
+
+set hive.optimize.point.lookup.min=2;
+
+explain
+select count(1)
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
+
+select '3 is expected:',count(1)
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_varchar.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_varchar.q b/ql/src/test/queries/clientpositive/in_typecheck_varchar.q
new file mode 100644
index 0000000..e8dbf46
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_varchar.q
@@ -0,0 +1,16 @@
+
+create table ax(s varchar(1),t varchar(10));
+
+insert into ax values ('a','a'),('a','a '),('b','bb');
+
+explain
+select 'expected 1',count(*) from ax where s = 'a' and t = 'a';
+select 'expected 1',count(*) from ax where s = 'a' and t = 'a';
+
+explain
+select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+
+select 'expected 0',count(*) from ax where t = 'a ';
+select 'expected 0',count(*) from ax where t = 'a ';
+select 'expected 0',count(*) from ax where t = 'a d';
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/pcs.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/pcs.q b/ql/src/test/queries/clientpositive/pcs.q
index 2e7eff4..31a5f0b 100644
--- a/ql/src/test/queries/clientpositive/pcs.q
+++ b/ql/src/test/queries/clientpositive/pcs.q
@@ -17,7 +17,6 @@ set hive.optimize.point.lookup.min = 1;
explain extended select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds;
select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds;
-set hive.optimize.point.lookup = false;
set hive.optimize.partition.columns.separate=true;
set hive.optimize.ppd=true;
@@ -27,7 +26,6 @@ select ds from pcs_t1 where struct(ds, key) in (struct('2000-04-08',1), struct('
explain extended select ds from pcs_t1 where struct(ds, key+2) in (struct('2000-04-08',3), struct('2000-04-09',4));
select ds from pcs_t1 where struct(ds, key+2) in (struct('2000-04-08',3), struct('2000-04-09',4));
-set hive.cbo.enable=false;
explain extended select /*+ MAPJOIN(pcs_t1) */ a.ds, b.key from pcs_t1 a join pcs_t1 b on a.ds=b.ds where struct(a.ds, a.key, b.ds) in (struct('2000-04-08',1, '2000-04-09'), struct('2000-04-09',2, '2000-04-08'));
select /*+ MAPJOIN(pcs_t1) */ a.ds, b.key from pcs_t1 a join pcs_t1 b on a.ds=b.ds where struct(a.ds, a.key, b.ds) in (struct('2000-04-08',1, '2000-04-09'), struct('2000-04-09',2, '2000-04-08'));
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/vector_struct_in.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_struct_in.q b/ql/src/test/queries/clientpositive/vector_struct_in.q
index 49a9374..ed03214 100644
--- a/ql/src/test/queries/clientpositive/vector_struct_in.q
+++ b/ql/src/test/queries/clientpositive/vector_struct_in.q
@@ -1,4 +1,4 @@
-set hive.cbo.enable=false;
+set hive.cbo.enable=true;
set hive.explain.user=false;
set hive.tez.dynamic.partition.pruning=false;
set hive.vectorized.execution.enabled=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientnegative/udf_in_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/udf_in_2.q.out b/ql/src/test/results/clientnegative/udf_in_2.q.out
deleted file mode 100644
index 83883e3..0000000
--- a/ql/src/test/results/clientnegative/udf_in_2.q.out
+++ /dev/null
@@ -1 +0,0 @@
-FAILED: SemanticException Line 0:-1 Wrong arguments 'false': The arguments for IN should be the same type! Types are: {int IN (boolean, boolean)}
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
index f6c3c56..d484f9e 100644
--- a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
+++ b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
@@ -160,7 +160,7 @@ POSTHOOK: Input: default@alter_coltype
#### A masked pattern was here ####
OPTIMIZED SQL: SELECT COUNT(*) AS `$f0`
FROM `default`.`alter_coltype`
-WHERE `ts` = 3 AND `dt` = 100
+WHERE `ts` = 3.0 AND `dt` = 100
STAGE DEPENDENCIES:
Stage-0 is a root stage
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
index a8b595a..b6e9077 100644
--- a/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
+++ b/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
@@ -57,10 +57,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src
- filterExpr: (((value) IN ('val_400', 'val_500') and (key) IN (400, 450)) or ((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200))) (type: boolean)
+ filterExpr: ((((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450))) or (((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean)
+ predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: string)
@@ -68,11 +68,12 @@ STAGE PLANS:
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: string)
+ Execution mode: vectorized
Reduce Operator Tree:
Forward
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean)
+ predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
@@ -108,7 +109,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Filter Operator
- predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean)
+ predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
@@ -582,7 +583,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean)
+ predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: key (type: string)
@@ -590,11 +591,12 @@ STAGE PLANS:
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: string)
+ Execution mode: vectorized
Reduce Operator Tree:
Forward
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean)
+ predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
@@ -630,7 +632,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Filter Operator
- predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean)
+ predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/in_typecheck_char.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/in_typecheck_char.q.out b/ql/src/test/results/clientpositive/in_typecheck_char.q.out
new file mode 100644
index 0000000..6948719
--- /dev/null
+++ b/ql/src/test/results/clientpositive/in_typecheck_char.q.out
@@ -0,0 +1,261 @@
+PREHOOK: query: create table ax(s char(1),t char(10))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ax
+POSTHOOK: query: create table ax(s char(1),t char(10))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ax
+PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ax
+POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ax
+POSTHOOK: Lineage: ax.s SCRIPT []
+POSTHOOK: Lineage: ax.t SCRIPT []
+PREHOOK: query: explain
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: ax
+ filterExpr: ((s = 'a') and (t = 'a ')) (type: boolean)
+ Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((s = 'a') and (t = 'a ')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 'expected 2' (type: string), _col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2 2
+PREHOOK: query: explain
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: ax
+ filterExpr: (struct(s,t)) IN (const struct('a','a '), const struct('b','bb ')) (type: boolean)
+ Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (struct(s,t)) IN (const struct('a','a '), const struct('b','bb ')) (type: boolean)
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 'expected 3' (type: string), _col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 3 3
+PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a '
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a '
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2 2
+PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a '
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a '
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2 2
+PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a d'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a d'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 0 0
+PREHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2 2
+PREHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: ax
+ filterExpr: (((s = 'a') and (t = 'a ')) or ((s = null) and (t = 'bb '))) is null (type: boolean)
+ Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((s = 'a') and (t = 'a ')) or ((s = null) and (t = 'bb '))) is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 'expected 1' (type: string), _col0 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 1 1
http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out b/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out
new file mode 100644
index 0000000..fb3f330
--- /dev/null
+++ b/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out
@@ -0,0 +1,99 @@
+PREHOOK: query: create table t (a string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (a string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values ('1'),('x'),('2.0')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values ('1'),('x'),('2.0')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a SCRIPT []
+PREHOOK: query: explain
+select * from t where a in (1.0,'x',2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t where a in (1.0,'x',2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: ((UDFToDouble(a)) IN (1.0D, 2.0D) or (a = 'x')) (type: boolean)
+ Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((UDFToDouble(a)) IN (1.0D, 2.0D) or (a = 'x')) (type: boolean)
+ Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from t where a in (1.0,'x',2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t where a in (1.0,'x',2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+1
+x
+2.0
+PREHOOK: query: create table ax(s char(1),t char(10))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ax
+POSTHOOK: query: create table ax(s char(1),t char(10))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ax
+PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ax
+POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ax
+POSTHOOK: Lineage: ax.s SCRIPT []
+POSTHOOK: Lineage: ax.t SCRIPT []
+PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 1 1