You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/11/03 16:30:50 UTC

[4/5] hive git commit: HIVE-20617: Fix type of constants in IN expressions to have correct type (Zoltan Haindrich via Jesus Camacho Rodriguez)

HIVE-20617: Fix type of constants in IN expressions to have correct type (Zoltan Haindrich via Jesus Camacho Rodriguez)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83d8d4f1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83d8d4f1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83d8d4f1

Branch: refs/heads/master
Commit: 83d8d4f159edee28497184047369159e2ef28b8b
Parents: f3fab45
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Sat Nov 3 09:28:41 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Sat Nov 3 09:28:41 2018 -0700

----------------------------------------------------------------------
 .../hadoop/hive/common/type/HiveChar.java       |   7 +
 .../org/apache/hadoop/hive/ql/ErrorMsg.java     |  15 +-
 .../rules/HivePointLookupOptimizerRule.java     | 249 +++++++++++-------
 .../hive/ql/parse/BaseSemanticAnalyzer.java     |   2 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |   5 +-
 .../hadoop/hive/ql/parse/TypeCheckCtx.java      |   7 +-
 .../hive/ql/parse/TypeCheckProcFactory.java     | 247 +++++++++++++-----
 .../ql/parse/TypeCheckProcFactoryUtils.java     | 144 ++++++++++
 .../hadoop/hive/ql/plan/ExprNodeDescUtils.java  |  35 ++-
 ql/src/test/queries/clientnegative/udf_in_2.q   |   2 -
 .../groupby_multi_single_reducer3.q             |   2 +
 .../queries/clientpositive/in_typecheck_char.q  |  24 ++
 .../queries/clientpositive/in_typecheck_mixed.q |  12 +
 .../clientpositive/in_typecheck_pointlook.q     |  35 +++
 .../clientpositive/in_typecheck_varchar.q       |  16 ++
 ql/src/test/queries/clientpositive/pcs.q        |   2 -
 .../queries/clientpositive/vector_struct_in.q   |   2 +-
 .../test/results/clientnegative/udf_in_2.q.out  |   1 -
 .../alter_partition_coltype.q.out               |   2 +-
 .../groupby_multi_single_reducer3.q.out         |  16 +-
 .../clientpositive/in_typecheck_char.q.out      | 261 +++++++++++++++++++
 .../clientpositive/in_typecheck_mixed.q.out     |  99 +++++++
 .../clientpositive/in_typecheck_pointlook.q.out | 181 +++++++++++++
 .../clientpositive/in_typecheck_varchar.q.out   | 185 +++++++++++++
 .../clientpositive/infer_const_type.q.out       |   8 +-
 ql/src/test/results/clientpositive/join45.q.out |   4 +-
 .../test/results/clientpositive/join45X.q.out   | 256 ++++++++++++++++++
 ql/src/test/results/clientpositive/join47.q.out |   4 +-
 .../clientpositive/llap/acid_no_buckets.q.out   |  32 +--
 .../clientpositive/llap/bucketpruning1.q.out    |   2 +-
 .../clientpositive/llap/check_constraint.q.out  |   6 +-
 .../results/clientpositive/llap/dec_str.q.out   |  10 +-
 .../llap/dynpart_sort_optimization_acid.q.out   |  49 ++--
 .../clientpositive/llap/explainuser_1.q.out     |  42 +--
 .../clientpositive/llap/is_distinct_from.q.out  |   2 +-
 .../test/results/clientpositive/llap/kryo.q.out |   2 +-
 .../results/clientpositive/llap/lineage3.q.out  |   2 +-
 .../clientpositive/llap/multi_column_in.q.out   |  16 +-
 .../llap/multi_column_in_single.q.out           |  16 +-
 .../clientpositive/llap/orc_llap_counters.q.out |  10 +-
 .../llap/orc_predicate_pushdown.q.out           |  26 +-
 .../llap/parquet_predicate_pushdown.q.out       |  26 +-
 .../clientpositive/llap/vector_between_in.q.out |  18 +-
 .../clientpositive/llap/vector_struct_in.q.out  |   6 +-
 .../clientpositive/llap/vectorization_13.q.out  |   8 +-
 .../clientpositive/llap/vectorization_6.q.out   |   4 +-
 .../clientpositive/llap/vectorization_8.q.out   |   8 +-
 .../llap/vectorization_short_regress.q.out      |   8 +-
 .../test/results/clientpositive/mapjoin47.q.out |   4 +-
 .../parquet_vectorization_13.q.out              |   8 +-
 .../parquet_vectorization_6.q.out               |   4 +-
 .../parquet_vectorization_8.q.out               |   8 +-
 ql/src/test/results/clientpositive/pcs.q.out    | 102 ++++----
 .../clientpositive/perf/tez/cbo_query15.q.out   |   2 +-
 .../perf/tez/constraints/cbo_query15.q.out      |   2 +-
 .../results/clientpositive/ppd_udf_col.q.out    |  10 +-
 .../spark/groupby_multi_single_reducer3.q.out   |  16 +-
 .../spark/parquet_vectorization_13.q.out        |   8 +-
 .../spark/parquet_vectorization_6.q.out         |   4 +-
 .../spark/parquet_vectorization_8.q.out         |   8 +-
 .../spark/spark_explainuser_1.q.out             |  42 +--
 .../spark/vector_between_in.q.out               |  18 +-
 .../clientpositive/spark/vectorization_13.q.out |   8 +-
 .../clientpositive/spark/vectorization_6.q.out  |   4 +-
 .../spark/vectorization_short_regress.q.out     |   8 +-
 .../vector_non_constant_in_expr.q.out           |   4 +-
 .../clientpositive/vector_struct_in.q.out       |   6 +-
 .../clientpositive/vectorization_13.q.out       |   8 +-
 .../clientpositive/vectorization_6.q.out        |   4 +-
 .../clientpositive/vectorization_8.q.out        |   8 +-
 70 files changed, 1926 insertions(+), 476 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
index 29dc06d..f0b28c7 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
@@ -43,6 +43,7 @@ public class HiveChar extends HiveBaseChar
   /**
    * Set char value, padding or truncating the value to the size of len parameter.
    */
+  @Override
   public void setValue(String val, int len) {
     super.setValue(HiveBaseChar.getPaddedValue(val, len), -1);
   }
@@ -59,15 +60,18 @@ public class HiveChar extends HiveBaseChar
     return value;
   }
 
+  @Override
   public int getCharacterLength() {
     String strippedValue = getStrippedValue();
     return strippedValue.codePointCount(0, strippedValue.length());
   }
 
+  @Override
   public String toString() {
     return getPaddedValue();
   }
 
+  @Override
   public int compareTo(HiveChar rhs) {
     if (rhs == this) {
       return 0;
@@ -75,6 +79,7 @@ public class HiveChar extends HiveBaseChar
     return this.getStrippedValue().compareTo(rhs.getStrippedValue());
   }
 
+  @Override
   public boolean equals(Object rhs) {
     if (rhs == this) {
       return true;
@@ -85,7 +90,9 @@ public class HiveChar extends HiveBaseChar
     return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue());
   }
 
+  @Override
   public int hashCode() {
     return getStrippedValue().hashCode();
   }
+
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index e7d7159..1a58311 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -18,13 +18,6 @@
 
 package org.apache.hadoop.hive.ql;
 
-import java.io.FileNotFoundException;
-import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import org.antlr.runtime.tree.Tree;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
@@ -35,6 +28,13 @@ import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin;
 import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
 import org.apache.hadoop.security.AccessControlException;
 
+import java.io.FileNotFoundException;
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 /**
  * List of all error messages.
  * This list contains both compile time and run-time errors.
@@ -469,6 +469,7 @@ public enum ErrorMsg {
   LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"),
   RESOURCE_PLAN_ALREADY_EXISTS(10417, "Resource plan {0} already exists", true),
   RESOURCE_PLAN_NOT_EXISTS(10418, "Resource plan {0} does not exist", true),
+  INCOMPATIBLE_STRUCT(10419, "Incompatible structs.", true),
 
   //========================== 20000 range starts here ========================//
 

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
index 04800cc..9a525ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
@@ -20,25 +20,24 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptRuleCall;
 import org.apache.calcite.plan.RelOptRuleOperand;
 import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.AbstractRelNode;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexShuttle;
 import org.apache.calcite.rex.RexUtil;
@@ -59,15 +58,18 @@ import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
 import com.google.common.collect.Sets;
 
-public abstract class HivePointLookupOptimizerRule extends RelOptRule {
-
 /**
- * This optimization will take a Filter or expression, and if its predicate contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
+ * This optimization attempts to identify and close expanded INs.
+ *
+ * Basically:
+ * <pre>
+ * (c) IN ( v1, v2, ...) &lt;=&gt; c1=v1 || c1=v2 || ...
+ * </pre>
+ * If c is struct; then c=v1 is a group of anded equations.
  */
+public abstract class HivePointLookupOptimizerRule extends RelOptRule {
+
+  /** Rule adapter to apply the transformation to Filter conditions. */
   public static class FilterCondition extends HivePointLookupOptimizerRule {
     public FilterCondition (int minNumORClauses) {
       super(operand(Filter.class, any()), minNumORClauses);
@@ -78,22 +80,20 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
       final Filter filter = call.rel(0);
       final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
       final RexNode condition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
-      analyzeCondition(call , rexBuilder, filter, condition);
-    }
 
-    @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
-      final Filter filter  = (Filter) node;
-      return filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
+      RexNode newCondition = analyzeRexNode(rexBuilder, condition);
+
+      // If we could not transform anything, we bail out
+      if (newCondition.toString().equals(condition.toString())) {
+        return;
+      }
+      RelNode newNode = filter.copy(filter.getTraitSet(), filter.getInput(), newCondition);
+
+      call.transformTo(newNode);
     }
   }
 
-/**
- * This optimization will take a Join or expression, and if its join condition contains
- * an OR operator whose children are constant equality expressions, it will try
- * to generate an IN clause (which is more efficient). If the OR operator contains
- * AND operator children, the optimization might generate an IN clause that uses
- * structs.
- */
+  /** Rule adapter to apply the transformation to Join conditions. */
   public static class JoinCondition extends HivePointLookupOptimizerRule {
     public JoinCondition (int minNumORClauses) {
       super(operand(Join.class, any()), minNumORClauses);
@@ -104,18 +104,55 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
       final Join join = call.rel(0);
       final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
       final RexNode condition = RexUtil.pullFactors(rexBuilder, join.getCondition());
-      analyzeCondition(call , rexBuilder, join, condition);
+
+      RexNode newCondition = analyzeRexNode(rexBuilder, condition);
+
+      // If we could not transform anything, we bail out
+      if (newCondition.toString().equals(condition.toString())) {
+        return;
+      }
+
+      RelNode newNode = join.copy(join.getTraitSet(),
+          newCondition,
+          join.getLeft(),
+          join.getRight(),
+          join.getJoinType(),
+          join.isSemiJoinDone());
+
+      call.transformTo(newNode);
+    }
+  }
+
+  /** Rule adapter to apply the transformation to Projections. */
+  public static class ProjectionExpressions extends HivePointLookupOptimizerRule {
+    public ProjectionExpressions(int minNumORClauses) {
+      super(operand(Project.class, any()), minNumORClauses);
     }
 
-    @Override protected RelNode copyNode(AbstractRelNode node, RexNode newCondition) {
-      final Join join = (Join) node;
-      return join.copy(join.getTraitSet(),
-              newCondition,
-              join.getLeft(),
-              join.getRight(),
-              join.getJoinType(),
-              join.isSemiJoinDone());
+    @Override
+    public void onMatch(RelOptRuleCall call) {
+      final Project project = call.rel(0);
+      boolean changed = false;
+      final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
+      List<RexNode> newProjects = new ArrayList<>();
+      for (RexNode oldNode : project.getProjects()) {
+        RexNode newNode = analyzeRexNode(rexBuilder, oldNode);
+        if (!newNode.toString().equals(oldNode.toString())) {
+          changed = true;
+          newProjects.add(newNode);
+        } else {
+          newProjects.add(oldNode);
+        }
+      }
+      if (!changed) {
+        return;
+      }
+      Project newProject = project.copy(project.getTraitSet(), project.getInput(), newProjects,
+          project.getRowType(), project.getFlags());
+      call.transformTo(newProject);
+
     }
+
   }
 
   protected static final Logger LOG = LoggerFactory.getLogger(HivePointLookupOptimizerRule.class);
@@ -123,37 +160,21 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
   // Minimum number of OR clauses needed to transform into IN clauses
   protected final int minNumORClauses;
 
-  protected abstract RelNode copyNode(AbstractRelNode node, RexNode newCondition);
-
   protected HivePointLookupOptimizerRule(
     RelOptRuleOperand operand, int minNumORClauses) {
     super(operand);
     this.minNumORClauses = minNumORClauses;
   }
 
-  public void analyzeCondition(RelOptRuleCall call,
-          RexBuilder rexBuilder,
-          AbstractRelNode node,
-          RexNode condition) {
-
+  public RexNode analyzeRexNode(RexBuilder rexBuilder, RexNode condition) {
     // 1. We try to transform possible candidates
-    RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, node,
-            minNumORClauses);
+    RexTransformIntoInClause transformIntoInClause = new RexTransformIntoInClause(rexBuilder, minNumORClauses);
     RexNode newCondition = transformIntoInClause.apply(condition);
 
     // 2. We merge IN expressions
     RexMergeInClause mergeInClause = new RexMergeInClause(rexBuilder);
     newCondition = mergeInClause.apply(newCondition);
-
-    // 3. If we could not transform anything, we bail out
-    if (newCondition.toString().equals(condition.toString())) {
-      return;
-    }
-
-    // 4. We create the Filter/Join with the new condition
-    RelNode newNode = copyNode(node, newCondition);
-
-    call.transformTo(newNode);
+    return newCondition;
   }
 
 
@@ -162,11 +183,9 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
    */
   protected static class RexTransformIntoInClause extends RexShuttle {
     private final RexBuilder rexBuilder;
-    private final AbstractRelNode nodeOp;
     private final int minNumORClauses;
 
-    RexTransformIntoInClause(RexBuilder rexBuilder, AbstractRelNode nodeOp, int minNumORClauses) {
-      this.nodeOp = nodeOp;
+    RexTransformIntoInClause(RexBuilder rexBuilder, int minNumORClauses) {
       this.rexBuilder = rexBuilder;
       this.minNumORClauses = minNumORClauses;
     }
@@ -180,7 +199,7 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
         case OR:
           try {
             RexNode newNode = transformIntoInClauseCondition(rexBuilder,
-                nodeOp.getRowType(), call, minNumORClauses);
+                call, minNumORClauses);
             if (newNode != null) {
               return newNode;
             }
@@ -196,18 +215,56 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
     }
 
     /**
-     * Represents a simple contraint.
+     * This class just wraps around a RexNode enables equals/hashCode based on toString.
+     *
+     * After CALCITE-2632 this might not be needed anymore */
+    static class RexNodeRef {
+
+      public static Comparator<RexNodeRef> COMPARATOR =
+          (RexNodeRef o1, RexNodeRef o2) -> o1.node.toString().compareTo(o2.node.toString());
+      private RexNode node;
+
+      public RexNodeRef(RexNode node) {
+        this.node = node;
+      }
+
+      public RexNode getRexNode() {
+        return node;
+      }
+
+      @Override
+      public int hashCode() {
+        return node.toString().hashCode();
+      }
+
+      @Override
+      public boolean equals(Object o) {
+        if (o instanceof RexNodeRef) {
+          RexNodeRef otherRef = (RexNodeRef) o;
+          return node.toString().equals(otherRef.node.toString());
+        }
+        return false;
+      }
+
+      @Override
+      public String toString() {
+        return "ref for:" + node.toString();
+      }
+    }
+    /**
+     * Represents a contraint.
      *
      * Example: a=1
+     * substr(a,1,2) = concat('asd','xxx')
      */
     static class Constraint {
 
-      private RexLiteral literal;
-      private RexInputRef inputRef;
+      private RexNode exprNode;
+      private RexNode constNode;
 
-      public Constraint(RexInputRef inputRef, RexLiteral literal) {
-        this.literal = literal;
-        this.inputRef = inputRef;
+      public Constraint(RexNode exprNode, RexNode constNode) {
+        this.exprNode = constNode;
+        this.constNode = exprNode;
       }
 
       /**
@@ -223,21 +280,31 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
         }
         RexNode opA = call.operands.get(0);
         RexNode opB = call.operands.get(1);
-        if (opA instanceof RexLiteral && opB instanceof RexInputRef) {
-          RexLiteral rexLiteral = (RexLiteral) opA;
-          RexInputRef rexInputRef = (RexInputRef) opB;
-          return new Constraint(rexInputRef, rexLiteral);
+        if (RexUtil.isNull(opA) || RexUtil.isNull(opB)) {
+          // dont try to compare nulls
+          return null;
+        }
+        if (isConstExpr(opA) && isColumnExpr(opB)) {
+          return new Constraint(opB, opA);
         }
-        if (opA instanceof RexInputRef && opB instanceof RexLiteral) {
-          RexLiteral rexLiteral = (RexLiteral) opB;
-          RexInputRef rexInputRef = (RexInputRef) opA;
-          return new Constraint(rexInputRef, rexLiteral);
+        if (isColumnExpr(opA) && isConstExpr(opB)) {
+          return new Constraint(opA, opB);
         }
         return null;
       }
 
-      public RexInputRef getKey() {
-        return inputRef;
+      private static boolean isColumnExpr(RexNode node) {
+        return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() > 0
+            && HiveCalciteUtil.isDeterministic(node);
+      }
+
+      private static boolean isConstExpr(RexNode node) {
+        return !node.getType().isStruct() && HiveCalciteUtil.getInputRefs(node).size() == 0
+            && HiveCalciteUtil.isDeterministic(node);
+      }
+
+      public RexNodeRef getKey() {
+        return new RexNodeRef(constNode);
       }
 
     }
@@ -254,17 +321,17 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
      *
      */
     static class ConstraintGroup {
+      public static final Function<ConstraintGroup, Set<RexNodeRef>> KEY_FUNCTION =
+          new Function<ConstraintGroup, Set<RexNodeRef>>() {
 
-      public static final Function<ConstraintGroup, Set<RexInputRef>> KEY_FUNCTION = new Function<ConstraintGroup, Set<RexInputRef>>() {
-
-        @Override
-        public Set<RexInputRef> apply(ConstraintGroup a) {
-          return a.key;
-        }
-      };
-      private Map<RexInputRef, Constraint> constraints = new HashMap<>();
+            @Override
+            public Set<RexNodeRef> apply(ConstraintGroup cg) {
+              return cg.key;
+            }
+          };
+      private Map<RexNodeRef, Constraint> constraints = new HashMap<>();
       private RexNode originalRexNode;
-      private final Set<RexInputRef> key;
+      private final Set<RexNodeRef> key;
 
       public ConstraintGroup(RexNode rexNode) {
         originalRexNode = rexNode;
@@ -289,21 +356,21 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
         key = constraints.keySet();
       }
 
-      public List<RexNode> getValuesInOrder(List<RexInputRef> columns) throws SemanticException {
+      public List<RexNode> getValuesInOrder(List<RexNodeRef> columns) throws SemanticException {
         List<RexNode> ret = new ArrayList<>();
-        for (RexInputRef rexInputRef : columns) {
+        for (RexNodeRef rexInputRef : columns) {
           Constraint constraint = constraints.get(rexInputRef);
           if (constraint == null) {
             throw new SemanticException("Unable to find constraint which was earlier added.");
           }
-          ret.add(constraint.literal);
+          ret.add(constraint.exprNode);
         }
         return ret;
       }
     }
 
-    private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RelDataType inputSchema,
-            RexNode condition, int minNumORClauses) throws SemanticException {
+    private RexNode transformIntoInClauseCondition(RexBuilder rexBuilder, RexNode condition,
+            int minNumORClauses) throws SemanticException {
       assert condition.getKind() == SqlKind.OR;
 
       ImmutableList<RexNode> operands = RexUtil.flattenOr(((RexCall) condition).getOperands());
@@ -318,10 +385,10 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
         allNodes.add(m);
       }
 
-      Multimap<Set<RexInputRef>, ConstraintGroup> assignmentGroups =
+      Multimap<Set<RexNodeRef>, ConstraintGroup> assignmentGroups =
           Multimaps.index(allNodes, ConstraintGroup.KEY_FUNCTION);
 
-      for (Entry<Set<RexInputRef>, Collection<ConstraintGroup>> sa : assignmentGroups.asMap().entrySet()) {
+      for (Entry<Set<RexNodeRef>, Collection<ConstraintGroup>> sa : assignmentGroups.asMap().entrySet()) {
         // skip opaque
         if (sa.getKey().size() == 0) {
           continue;
@@ -351,13 +418,15 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule {
 
     }
 
-    private RexNode buildInFor(Set<RexInputRef> set, Collection<ConstraintGroup> value) throws SemanticException {
+    private RexNode buildInFor(Set<RexNodeRef> set, Collection<ConstraintGroup> value) throws SemanticException {
 
-      List<RexInputRef> columns = new ArrayList<RexInputRef>();
+      List<RexNodeRef> columns = new ArrayList<>();
       columns.addAll(set);
+      columns.sort(RexNodeRef.COMPARATOR);
       List<RexNode >operands = new ArrayList<>();
 
-      operands.add(useStructIfNeeded(columns));
+      List<RexNode> columnNodes = columns.stream().map(n -> n.getRexNode()).collect(Collectors.toList());
+      operands.add(useStructIfNeeded(columnNodes));
       for (ConstraintGroup node : value) {
         List<RexNode> values = node.getValuesInOrder(columns);
         operands.add(useStructIfNeeded(values));

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index c9df668..3a51d97 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -910,7 +910,7 @@ public abstract class BaseSemanticAnalyzer {
       } catch(Exception e) {
         throw new SemanticException(
             ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("Invalid CHECK constraint expression: ")
-                + cc.getCheck_expression() + ". " + e.getMessage());
+                + cc.getCheck_expression() + ". " + e.getMessage(), e);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 82e975a..d92cfca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
+
 import org.antlr.runtime.ClassicToken;
 import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.tree.Tree;
@@ -277,7 +278,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.joda.time.Interval;
 
-import javax.sql.DataSource;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
@@ -304,6 +304,8 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import javax.sql.DataSource;
+
 
 public class CalcitePlanner extends SemanticAnalyzer {
 
@@ -2064,6 +2066,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
       if (conf.getBoolVar(HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {
         rules.add(new HivePointLookupOptimizerRule.FilterCondition(minNumORClauses));
         rules.add(new HivePointLookupOptimizerRule.JoinCondition(minNumORClauses));
+        rules.add(new HivePointLookupOptimizerRule.ProjectionExpressions(minNumORClauses));
       }
       if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_CONSTRAINTS_JOIN) &&
           profilesCBO.contains(ExtendedCBOProfile.REFERENTIAL_CONSTRAINTS)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
index 1b56ecd..12e7ae6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
@@ -19,9 +19,10 @@
 package org.apache.hadoop.hive.ql.parse;
 
 import org.apache.calcite.rel.RelNode;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
 import java.util.Map;
 
 /**
@@ -263,4 +264,8 @@ public class TypeCheckCtx implements NodeProcessorCtx {
   public boolean isFoldExpr() {
     return foldExpr;
   }
+
+  public boolean isCBOExecuted() {
+    return foldExpr;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index 4968d16..fd151b9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
@@ -353,14 +354,18 @@ public class TypeCheckProcFactory {
     }
 
     public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) {
-      // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
-      //       precision and scale of the value by throwing away trailing zeroes. This may or may
-      //       not be desirable for the literals; however, this used to be the default behavior
-      //       for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
       HiveDecimal hd = HiveDecimal.create(strVal);
       if (notNull && hd == null) {
         return null;
       }
+      return new ExprNodeConstantDesc(adjustType(hd), hd);
+    }
+
+    private static DecimalTypeInfo adjustType(HiveDecimal hd) {
+      // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+      //       precision and scale of the value by throwing away trailing zeroes. This may or may
+      //       not be desirable for the literals; however, this used to be the default behavior
+      //       for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
       int prec = 1;
       int scale = 0;
       if (hd != null) {
@@ -368,7 +373,7 @@ public class TypeCheckProcFactory {
         scale = hd.scale();
       }
       DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
-      return new ExprNodeConstantDesc(typeInfo, hd);
+      return typeInfo;
     }
 
   }
@@ -1163,36 +1168,59 @@ public class TypeCheckProcFactory {
           ExprNodeDesc constChild = children.get(constIdx);
           ExprNodeDesc columnChild = children.get(1 - constIdx);
 
-            final PrimitiveTypeInfo colTypeInfo =
-                TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase());
-            ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild);
-            if (newChild == null) {
-              // non-interpretabe as that type...
-              if (genericUDF instanceof GenericUDFOPEqual) {
-                return new ExprNodeConstantDesc(false);
-              }
-            } else {
-              children.set(constIdx, newChild);
+          final PrimitiveTypeInfo colTypeInfo =
+              TypeInfoFactory.getPrimitiveTypeInfo(columnChild.getTypeString().toLowerCase());
+          ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, constChild);
+          if (newChild == null) {
+            // non-interpretable as target type...
+            // TODO: all comparisons with null should result in null
+            if (genericUDF instanceof GenericUDFOPEqual
+                && !(genericUDF instanceof GenericUDFOPEqualNS)) {
+              return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null);
             }
+          } else {
+            children.set(constIdx, newChild);
+          }
         }
-        if (genericUDF instanceof GenericUDFIn && children.get(0) instanceof ExprNodeColumnDesc) {
-          ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) children.get(0);
-          final PrimitiveTypeInfo colTypeInfo =
-              TypeInfoFactory.getPrimitiveTypeInfo(columnDesc.getTypeString().toLowerCase());
+        if (genericUDF instanceof GenericUDFIn) {
+
+          ExprNodeDesc columnDesc = children.get(0);
           List<ExprNodeDesc> outputOpList = children.subList(1, children.size());
           ArrayList<ExprNodeDesc> inOperands = new ArrayList<>(outputOpList);
           outputOpList.clear();
 
+          boolean hasNullValue = false;
           for (ExprNodeDesc oldChild : inOperands) {
-            if(oldChild !=null && oldChild instanceof ExprNodeConstantDesc) {
-              ExprNodeDesc newChild = interpretNodeAs(colTypeInfo, oldChild);
-              if(newChild == null) {
-                // non interpretable as target type; skip
-                continue;
+            if (oldChild == null) {
+              hasNullValue = true;
+              continue;
+            }
+            ExprNodeDesc newChild = interpretNodeAsStruct(columnDesc, oldChild);
+            if (newChild == null) {
+              hasNullValue = true;
+              continue;
+            }
+            outputOpList.add(newChild);
+          }
+
+          if (hasNullValue) {
+            ExprNodeConstantDesc nullConst = new ExprNodeConstantDesc(columnDesc.getTypeInfo(), null);
+            if (outputOpList.size() == 0) {
+              // we have found only null values...remove the IN ; it will be null all the time.
+              return nullConst;
+            }
+            outputOpList.add(nullConst);
+          }
+          if (!ctx.isCBOExecuted()) {
+            ArrayList<ExprNodeDesc> orOperands = TypeCheckProcFactoryUtils.rewriteInToOR(children);
+            if (orOperands != null) {
+              if (orOperands.size() == 1) {
+                orOperands.add(new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, false));
               }
-              outputOpList.add(newChild);
-            }else{
-              outputOpList.add(oldChild);
+              funcText = "or";
+              genericUDF = new GenericUDFOPOr();
+              children.clear();
+              children.addAll(orOperands);
             }
           }
         }
@@ -1258,48 +1286,145 @@ public class TypeCheckProcFactory {
       return desc;
     }
 
-    private ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) {
+    /**
+     * Interprets the given value as columnDesc if possible
+     */
+    private static ExprNodeDesc interpretNodeAsStruct(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc)
+        throws SemanticException {
+      if(columnDesc instanceof ExprNodeColumnDesc) {
+        ExprNodeColumnDesc exprNodeColumnDesc = (ExprNodeColumnDesc) columnDesc;
+        final PrimitiveTypeInfo typeInfo =
+            TypeInfoFactory.getPrimitiveTypeInfo(exprNodeColumnDesc.getTypeString().toLowerCase());
+        return interpretNodeAs(typeInfo, valueDesc);
+      }
+      if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isConstantStruct(valueDesc)) {
+        List<ExprNodeDesc> columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren();
+        ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc;
+        StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo();
+        ArrayList<TypeInfo> structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos();
+        ArrayList<TypeInfo> newStructFieldInfos = new ArrayList<>();
+
+        if (columnChilds.size() != structFieldInfos.size()) {
+          throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + structFieldInfos));
+        }
+        List<Object> oldValues = (List<Object>) valueConstDesc.getValue();
+        List<Object> newValues = new ArrayList<>();
+        for (int i = 0; i < columnChilds.size(); i++) {
+          newStructFieldInfos.add(columnChilds.get(i).getTypeInfo());
+          Object newValue = interpretConstantAsPrimitive(
+              (PrimitiveTypeInfo) columnChilds.get(i).getTypeInfo(),
+              oldValues.get(i),
+              structFieldInfos.get(i));
+          newValues.add(newValue);
+        }
+        StructTypeInfo sti = new StructTypeInfo();
+        sti.setAllStructFieldTypeInfos(newStructFieldInfos);
+        sti.setAllStructFieldNames(structTypeInfo.getAllStructFieldNames());
+        return new ExprNodeConstantDesc(sti, newValues);
+
+      }
+      if (ExprNodeDescUtils.isStructUDF(columnDesc) && ExprNodeDescUtils.isStructUDF(valueDesc)) {
+        List<ExprNodeDesc> columnChilds = ((ExprNodeGenericFuncDesc) columnDesc).getChildren();
+        List<ExprNodeDesc> valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren();
+        if (columnChilds.size() != valueChilds.size()) {
+          throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnChilds + " and " + valueChilds));
+        }
+        List<ExprNodeDesc> oldValueChilds = new ArrayList<>(valueChilds);
+        valueChilds.clear();
+        for (int i = 0; i < oldValueChilds.size(); i++) {
+          ExprNodeDesc newValue = interpretNodeAsStruct(columnChilds.get(i), oldValueChilds.get(i));
+          valueChilds.add(newValue);
+        }
+      }
+      return valueDesc;
+    }
+
+    private static ExprNodeDesc interpretNodeAs(PrimitiveTypeInfo colTypeInfo, ExprNodeDesc constChild) {
       if (constChild instanceof ExprNodeConstantDesc) {
         // Try to narrow type of constant
         Object constVal = ((ExprNodeConstantDesc) constChild).getValue();
-        String constType = constChild.getTypeString().toLowerCase();
-        if (constVal instanceof Number || constVal instanceof String) {
-          try {
-            PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry();
-            if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
-              return new ExprNodeConstantDesc(new Integer(constVal.toString()));
-            } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
-              return new ExprNodeConstantDesc(new Long(constVal.toString()));
-            } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
-              return new ExprNodeConstantDesc(new Double(constVal.toString()));
-            } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
-              return new ExprNodeConstantDesc(new Float(constVal.toString()));
-            } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
-              return new ExprNodeConstantDesc(new Byte(constVal.toString()));
-            } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
-              return new ExprNodeConstantDesc(new Short(constVal.toString()));
-            } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
-              return NumExprProcessor.createDecimal(constVal.toString(), false);
-            }
-          } catch (NumberFormatException nfe) {
-            LOG.trace("Failed to narrow type of constant", nfe);
-            if (!NumberUtils.isNumber(constVal.toString())) {
-              return null;
-            }
+        if (constVal == null) {
+          // adjust type of null
+          return new ExprNodeConstantDesc(colTypeInfo, null);
+        }
+        Object newConst = interpretConstantAsPrimitive(colTypeInfo, constVal, constChild.getTypeInfo());
+        if (newConst == null) {
+          return null;
+        }
+        if(newConst == constVal) {
+          return constChild;
+        } else {
+          return new ExprNodeConstantDesc(adjustType(colTypeInfo, newConst), newConst);
+        }
+      }
+      return constChild;
+    }
+
+    private static TypeInfo adjustType(PrimitiveTypeInfo colTypeInfo, Object newConst) {
+      if (newConst instanceof HiveDecimal) {
+        return NumExprProcessor.adjustType((HiveDecimal) newConst);
+      }
+      return colTypeInfo;
+    }
+
+    private static Object interpretConstantAsPrimitive(PrimitiveTypeInfo colTypeInfo, Object constVal,
+        TypeInfo constTypeInfo) {
+      String constTypeInfoName = constTypeInfo.getTypeName();
+      if (constVal instanceof Number || constVal instanceof String) {
+        try {
+          PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry();
+          if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+            return (new Integer(constVal.toString()));
+          } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+            return (new Long(constVal.toString()));
+          } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+            return (new Double(constVal.toString()));
+          } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+            return (new Float(constVal.toString()));
+          } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+            return (new Byte(constVal.toString()));
+          } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+            return (new Short(constVal.toString()));
+          } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+            return HiveDecimal.create(constVal.toString());
+          }
+        } catch (NumberFormatException nfe) {
+          LOG.trace("Failed to narrow type of constant", nfe);
+          if (!NumberUtils.isNumber(constVal.toString())) {
+            return null;
           }
         }
+      }
+
+      // Comparision of decimal and float/double happens in float/double.
+      if (constVal instanceof HiveDecimal) {
+        HiveDecimal hiveDecimal = (HiveDecimal) constVal;
 
-        // if column type is char and constant type is string, then convert the constant to char
-        // type with padded spaces.
-        if (constType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) {
-          final Object originalValue = ((ExprNodeConstantDesc) constChild).getValue();
-          final String constValue = originalValue.toString();
-          final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo);
-          final HiveChar newValue = new HiveChar(constValue, length);
-          return new ExprNodeConstantDesc(colTypeInfo, newValue);
+        PrimitiveTypeEntry primitiveTypeEntry = colTypeInfo.getPrimitiveTypeEntry();
+        if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+          return hiveDecimal.doubleValue();
+        } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+          return hiveDecimal.floatValue();
         }
+        return hiveDecimal;
       }
-      return constChild;
+
+      // TODO : Char and string comparison happens in char. But, varchar and string comparison happens in String.
+
+      // if column type is char and constant type is string, then convert the constant to char
+      // type with padded spaces.
+      if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && colTypeInfo instanceof CharTypeInfo) {
+        final String constValue = constVal.toString();
+        final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo);
+        HiveChar newValue = new HiveChar(constValue, length);
+        HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+        if (maxCharConst.equals(newValue)) {
+          return newValue;
+        } else {
+          return null;
+        }
+      }
+      return constVal;
     }
 
     private boolean canConvertIntoNvl(GenericUDF genericUDF, ArrayList<ExprNodeDesc> children) {

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java
new file mode 100644
index 0000000..b0544f3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactoryUtils.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
+import com.google.common.collect.Lists;
+
+public class TypeCheckProcFactoryUtils {
+
+  static ArrayList<ExprNodeDesc> rewriteInToOR(ArrayList<ExprNodeDesc> inOperands) throws SemanticException {
+    ExprNodeDesc columnDesc = inOperands.get(0);
+
+    ArrayList<ExprNodeDesc> orOperands = new ArrayList<>();
+    for (int i = 1; i < inOperands.size(); i++) {
+      ExprNodeDesc andExpr = buildEqualsArr(columnDesc, inOperands.get(i));
+      if (andExpr == null) {
+        return null;
+      }
+      orOperands.add(andExpr);
+    }
+    return orOperands;
+  }
+
+  private static ExprNodeDesc buildEqualsArr(ExprNodeDesc columnDesc, ExprNodeDesc exprNodeDesc)
+      throws SemanticException {
+    List<ExprNodeDesc> lNodes = asListOfNodes(columnDesc);
+    List<ExprNodeDesc> rNodes = asListOfNodes(exprNodeDesc);
+    if (lNodes == null || rNodes == null) {
+      // something went wrong
+      return null;
+    }
+    if (lNodes.size() != rNodes.size()) {
+      throw new SemanticException(ErrorMsg.INCOMPATIBLE_STRUCT.getMsg(columnDesc + " and " + exprNodeDesc));
+    }
+
+    List<ExprNodeDesc> ret = new ArrayList<>();
+    for (int i = 0; i < lNodes.size(); i++) {
+      ret.add(buildEquals(lNodes.get(i), rNodes.get(i)));
+    }
+    return buildAnd(ret);
+  }
+
+  private static ExprNodeGenericFuncDesc buildEquals(ExprNodeDesc columnDesc, ExprNodeDesc valueDesc) {
+    return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPEqual(), "=",
+        Lists.newArrayList(columnDesc, valueDesc));
+  }
+
+  private static ExprNodeDesc buildAnd(List<ExprNodeDesc> values) {
+    if (values.size() == 1) {
+      return values.get(0);
+    } else {
+      return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(), "and", values);
+    }
+  }
+
+  private static List<ExprNodeDesc> asListOfNodes(ExprNodeDesc desc) {
+    ExprNodeDesc valueDesc = desc;
+    if (ExprNodeDescUtils.isStructUDF(desc)) {
+      List<ExprNodeDesc> valueChilds = ((ExprNodeGenericFuncDesc) valueDesc).getChildren();
+      for (ExprNodeDesc exprNodeDesc : valueChilds) {
+        if (!isSafeExpression(exprNodeDesc)) {
+          return null;
+        }
+      }
+      return valueChilds;
+    }
+    if (ExprNodeDescUtils.isConstantStruct(valueDesc)) {
+      ExprNodeConstantDesc valueConstDesc = (ExprNodeConstantDesc) valueDesc;
+      List<Object> oldValues = (List<Object>) valueConstDesc.getValue();
+      StructTypeInfo structTypeInfo = (StructTypeInfo) valueConstDesc.getTypeInfo();
+      ArrayList<TypeInfo> structFieldInfos = structTypeInfo.getAllStructFieldTypeInfos();
+
+      List<ExprNodeDesc> ret = new ArrayList<>();
+      for (int i = 0; i < oldValues.size(); i++) {
+        ret.add(new ExprNodeConstantDesc(structFieldInfos.get(i), oldValues.get(i)));
+      }
+      return ret;
+    }
+    if (isSafeExpression(desc)) {
+      return Lists.newArrayList(desc);
+    }
+
+    return null;
+  }
+
+  private static boolean isSafeExpression(ExprNodeDesc desc) {
+    TypeInfo typeInfo = desc.getTypeInfo();
+    if (typeInfo.getCategory() != Category.PRIMITIVE) {
+      return false;
+    }
+    if (isConstantOrColumn(desc)) {
+      return true;
+    }
+    if (desc instanceof ExprNodeGenericFuncDesc) {
+      ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) desc;
+      if (FunctionRegistry.isConsistentWithinQuery(exprNodeGenericFuncDesc.getGenericUDF())) {
+        for (ExprNodeDesc child : exprNodeGenericFuncDesc.getChildren()) {
+          if (!isSafeExpression(child)) {
+            return false;
+          }
+        }
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean isConstantOrColumn(ExprNodeDesc desc) {
+    return desc instanceof ExprNodeColumnDesc || desc instanceof ExprNodeConstantDesc;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
index c274fd7..596640e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
@@ -18,18 +18,14 @@
 
 package org.apache.hadoop.hive.ql.plan;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
 import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -38,6 +34,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -45,10 +42,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.Pr
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.util.ReflectionUtils;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
 
 public class ExprNodeDescUtils {
 
@@ -571,7 +573,7 @@ public class ExprNodeDescUtils {
 
 	/**
 	 * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc.
-	 * 
+	 *
 	 * @param exprDesc
 	 * @param hashCodeToColumnDescMap
 	 *            Assumption: If two ExprNodeColumnDesc have same hash code then
@@ -642,7 +644,9 @@ public class ExprNodeDescUtils {
     // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
     // We also don't do anything for non-primitive children (maybe we should assert).
     if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
-        || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
+        || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) {
+      return pti;
+    }
     PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
     // If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
     return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
@@ -652,7 +656,7 @@ public class ExprNodeDescUtils {
    * Build ExprNodeColumnDesc for the projections in the input operator from
    * sartpos to endpos(both included). Operator must have an associated
    * colExprMap.
-   * 
+   *
    * @param inputOp
    *          Input Hive Operator
    * @param startPos
@@ -684,7 +688,7 @@ public class ExprNodeDescUtils {
     }
 
     return exprColLst;
-  }  
+  }
 
   public static List<ExprNodeDesc> flattenExprList(List<ExprNodeDesc> sourceList) {
     ArrayList<ExprNodeDesc> result = new ArrayList<ExprNodeDesc>(sourceList.size());
@@ -998,4 +1002,17 @@ public class ExprNodeDescUtils {
     }
     return false;
   }
+
+  public static boolean isConstantStruct(ExprNodeDesc valueDesc) {
+    return valueDesc instanceof ExprNodeConstantDesc && valueDesc.getTypeInfo() instanceof StructTypeInfo;
+  }
+
+  public static boolean isStructUDF(ExprNodeDesc columnDesc) {
+    if (columnDesc instanceof ExprNodeGenericFuncDesc) {
+      ExprNodeGenericFuncDesc exprNodeGenericFuncDesc = (ExprNodeGenericFuncDesc) columnDesc;
+      return (exprNodeGenericFuncDesc.getGenericUDF() instanceof GenericUDFStruct);
+    }
+    return false;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientnegative/udf_in_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/udf_in_2.q b/ql/src/test/queries/clientnegative/udf_in_2.q
deleted file mode 100644
index 2288054..0000000
--- a/ql/src/test/queries/clientnegative/udf_in_2.q
+++ /dev/null
@@ -1,2 +0,0 @@
-select 1=1 in (true, false);
-

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q b/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
index 94aea35..03af1cc 100644
--- a/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
+++ b/ql/src/test/queries/clientpositive/groupby_multi_single_reducer3.q
@@ -3,6 +3,8 @@
 
 -- SORT_QUERY_RESULTS
 
+set hive.cbo.enable=true;
+
 create table e1_n1 (key string, count int);
 create table e2_n2 (key string, count int);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_char.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_char.q b/ql/src/test/queries/clientpositive/in_typecheck_char.q
new file mode 100644
index 0000000..3955c4b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_char.q
@@ -0,0 +1,24 @@
+
+create table ax(s char(1),t char(10));
+
+insert into ax values ('a','a'),('a','a '),('b','bb');
+
+explain
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a';
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a';
+
+explain
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+
+select 'expected 2',count(*) from ax where t = 'a         ';
+select 'expected 2',count(*) from ax where t = 'a          ';
+select 'expected 0',count(*) from ax where t = 'a          d';
+
+
+select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb'));
+
+
+-- this is right now broken; HIVE-20779 should fix it
+explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null;
+select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null;

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_mixed.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_mixed.q b/ql/src/test/queries/clientpositive/in_typecheck_mixed.q
new file mode 100644
index 0000000..26c486b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_mixed.q
@@ -0,0 +1,12 @@
+create table t (a string);
+
+insert into t values ('1'),('x'),('2.0');
+
+explain
+select * from t where a in (1.0,'x',2);
+select * from t where a in (1.0,'x',2);
+
+
+create table ax(s char(1),t char(10));
+insert into ax values ('a','a'),('a','a '),('b','bb');
+select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null;

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q b/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q
new file mode 100644
index 0000000..c788506
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_pointlook.q
@@ -0,0 +1,35 @@
+
+create table customer_demographics (cd_marital_status char(1), cd_education_status char(20));
+
+insert into customer_demographics values
+('M','Unknown'),
+('W','Advanced Degree'),
+('W','Advanced Degree '),
+('W',' Advanced Degree')
+;
+
+set hive.optimize.point.lookup.min=32;
+
+explain
+select count(1)
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or      (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
+
+select '3 is expected:',count(1) 
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or      (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
+
+set hive.optimize.point.lookup.min=2;
+
+explain
+select count(1)
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or      (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));
+
+select '3 is expected:',count(1) 
+from customer_demographics
+where ( (cd_marital_status = 'M' and cd_education_status = 'Unknown')
+or      (cd_marital_status = 'W' and cd_education_status = 'Advanced Degree'));

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/in_typecheck_varchar.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/in_typecheck_varchar.q b/ql/src/test/queries/clientpositive/in_typecheck_varchar.q
new file mode 100644
index 0000000..e8dbf46
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/in_typecheck_varchar.q
@@ -0,0 +1,16 @@
+
+create table ax(s varchar(1),t varchar(10));
+
+insert into ax values ('a','a'),('a','a '),('b','bb');
+
+explain
+select 'expected 1',count(*) from ax where s = 'a' and t = 'a';
+select 'expected 1',count(*) from ax where s = 'a' and t = 'a';
+
+explain
+select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+select 'expected 2',count(*) from ax where (s,t) in (('a','a'),('b','bb'));
+
+select 'expected 0',count(*) from ax where t = 'a         ';
+select 'expected 0',count(*) from ax where t = 'a          ';
+select 'expected 0',count(*) from ax where t = 'a          d';

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/pcs.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/pcs.q b/ql/src/test/queries/clientpositive/pcs.q
index 2e7eff4..31a5f0b 100644
--- a/ql/src/test/queries/clientpositive/pcs.q
+++ b/ql/src/test/queries/clientpositive/pcs.q
@@ -17,7 +17,6 @@ set hive.optimize.point.lookup.min = 1;
 explain extended select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds;
 select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds;
 
-set hive.optimize.point.lookup = false;
 set hive.optimize.partition.columns.separate=true;
 set hive.optimize.ppd=true;
 
@@ -27,7 +26,6 @@ select ds from pcs_t1 where struct(ds, key) in (struct('2000-04-08',1), struct('
 explain extended select ds from pcs_t1 where struct(ds, key+2) in (struct('2000-04-08',3), struct('2000-04-09',4));
 select ds from pcs_t1 where struct(ds, key+2) in (struct('2000-04-08',3), struct('2000-04-09',4));
 
-set hive.cbo.enable=false;
 explain extended select /*+ MAPJOIN(pcs_t1) */ a.ds, b.key from pcs_t1 a join pcs_t1 b  on a.ds=b.ds where struct(a.ds, a.key, b.ds) in (struct('2000-04-08',1, '2000-04-09'), struct('2000-04-09',2, '2000-04-08'));
 
 select /*+ MAPJOIN(pcs_t1) */ a.ds, b.key from pcs_t1 a join pcs_t1 b  on a.ds=b.ds where struct(a.ds, a.key, b.ds) in (struct('2000-04-08',1, '2000-04-09'), struct('2000-04-09',2, '2000-04-08'));

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/queries/clientpositive/vector_struct_in.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_struct_in.q b/ql/src/test/queries/clientpositive/vector_struct_in.q
index 49a9374..ed03214 100644
--- a/ql/src/test/queries/clientpositive/vector_struct_in.q
+++ b/ql/src/test/queries/clientpositive/vector_struct_in.q
@@ -1,4 +1,4 @@
-set hive.cbo.enable=false;
+set hive.cbo.enable=true;
 set hive.explain.user=false;
 set hive.tez.dynamic.partition.pruning=false;
 set hive.vectorized.execution.enabled=true;

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientnegative/udf_in_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/udf_in_2.q.out b/ql/src/test/results/clientnegative/udf_in_2.q.out
deleted file mode 100644
index 83883e3..0000000
--- a/ql/src/test/results/clientnegative/udf_in_2.q.out
+++ /dev/null
@@ -1 +0,0 @@
-FAILED: SemanticException Line 0:-1 Wrong arguments 'false': The arguments for IN should be the same type! Types are: {int IN (boolean, boolean)}

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
index f6c3c56..d484f9e 100644
--- a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
+++ b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
@@ -160,7 +160,7 @@ POSTHOOK: Input: default@alter_coltype
 #### A masked pattern was here ####
 OPTIMIZED SQL: SELECT COUNT(*) AS `$f0`
 FROM `default`.`alter_coltype`
-WHERE `ts` = 3 AND `dt` = 100
+WHERE `ts` = 3.0 AND `dt` = 100
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
 

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
index a8b595a..b6e9077 100644
--- a/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
+++ b/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
@@ -57,10 +57,10 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: src
-            filterExpr: (((value) IN ('val_400', 'val_500') and (key) IN (400, 450)) or ((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200))) (type: boolean)
+            filterExpr: ((((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450))) or (((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200)))) (type: boolean)
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean)
+              predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean)
               Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: key (type: string)
@@ -68,11 +68,12 @@ STAGE PLANS:
                 Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
+      Execution mode: vectorized
       Reduce Operator Tree:
         Forward
           Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean)
+            predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean)
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               aggregations: count()
@@ -108,7 +109,7 @@ STAGE PLANS:
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
           Filter Operator
-            predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean)
+            predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean)
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               aggregations: count()
@@ -582,7 +583,7 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: (((value) IN ('val_100', 'val_200', 'val_300') and (key) IN (100, 150, 200)) or ((value) IN ('val_400', 'val_500') and (key) IN (400, 450))) (type: boolean)
+              predicate: ((((value = 'val_100') or (value = 'val_200') or (value = 'val_300')) and ((key = 100) or (key = 150) or (key = 200))) or (((value = 'val_400') or (value = 'val_500')) and ((key = 400) or (key = 450)))) (type: boolean)
               Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: key (type: string)
@@ -590,11 +591,12 @@ STAGE PLANS:
                 Map-reduce partition columns: key (type: string)
                 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
+      Execution mode: vectorized
       Reduce Operator Tree:
         Forward
           Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((KEY._col0) IN (100, 150, 200) and (VALUE._col0) IN ('val_100', 'val_200', 'val_300')) (type: boolean)
+            predicate: (((KEY._col0 = 100) or (KEY._col0 = 150) or (KEY._col0 = 200)) and ((VALUE._col0 = 'val_100') or (VALUE._col0 = 'val_200') or (VALUE._col0 = 'val_300'))) (type: boolean)
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               aggregations: count()
@@ -630,7 +632,7 @@ STAGE PLANS:
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
           Filter Operator
-            predicate: ((KEY._col0) IN (400, 450) and (VALUE._col0) IN ('val_400', 'val_500')) (type: boolean)
+            predicate: (((KEY._col0 = 400) or (KEY._col0 = 450)) and ((VALUE._col0 = 'val_400') or (VALUE._col0 = 'val_500'))) (type: boolean)
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               aggregations: count()

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/in_typecheck_char.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/in_typecheck_char.q.out b/ql/src/test/results/clientpositive/in_typecheck_char.q.out
new file mode 100644
index 0000000..6948719
--- /dev/null
+++ b/ql/src/test/results/clientpositive/in_typecheck_char.q.out
@@ -0,0 +1,261 @@
+PREHOOK: query: create table ax(s char(1),t char(10))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ax
+POSTHOOK: query: create table ax(s char(1),t char(10))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ax
+PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ax
+POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ax
+POSTHOOK: Lineage: ax.s SCRIPT []
+POSTHOOK: Lineage: ax.t SCRIPT []
+PREHOOK: query: explain
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: ax
+            filterExpr: ((s = 'a') and (t = 'a         ')) (type: boolean)
+            Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: ((s = 'a') and (t = 'a         ')) (type: boolean)
+              Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+                Group By Operator
+                  aggregations: count()
+                  mode: hash
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    sort order: 
+                    Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+                    value expressions: _col0 (type: bigint)
+      Execution mode: vectorized
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: count(VALUE._col0)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: 'expected 2' (type: string), _col0 (type: bigint)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where s = 'a' and t = 'a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2	2
+PREHOOK: query: explain
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: ax
+            filterExpr: (struct(s,t)) IN (const struct('a','a         '), const struct('b','bb        ')) (type: boolean)
+            Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (struct(s,t)) IN (const struct('a','a         '), const struct('b','bb        ')) (type: boolean)
+              Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+                Group By Operator
+                  aggregations: count()
+                  mode: hash
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    sort order: 
+                    Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+                    value expressions: _col0 (type: bigint)
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: count(VALUE._col0)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: 'expected 3' (type: string), _col0 (type: bigint)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 3',count(*) from ax where (s,t) in (('a','a'),('b','bb'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 3	3
+PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a         '
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a         '
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2	2
+PREHOOK: query: select 'expected 2',count(*) from ax where t = 'a          '
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where t = 'a          '
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2	2
+PREHOOK: query: select 'expected 0',count(*) from ax where t = 'a          d'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 0',count(*) from ax where t = 'a          d'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 0	0
+PREHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb'))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 2',count(*) from ax where (s,t) in (('a','a'),(null, 'bb'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 2	2
+PREHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: explain select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: ax
+            filterExpr: (((s = 'a') and (t = 'a         ')) or ((s = null) and (t = 'bb        '))) is null (type: boolean)
+            Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (((s = 'a') and (t = 'a         ')) or ((s = null) and (t = 'bb        '))) is null (type: boolean)
+              Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+                Group By Operator
+                  aggregations: count()
+                  mode: hash
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    sort order: 
+                    Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+                    value expressions: _col0 (type: bigint)
+      Execution mode: vectorized
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: count(VALUE._col0)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: 'expected 1' (type: string), _col0 (type: bigint)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 1	1

http://git-wip-us.apache.org/repos/asf/hive/blob/83d8d4f1/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out b/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out
new file mode 100644
index 0000000..fb3f330
--- /dev/null
+++ b/ql/src/test/results/clientpositive/in_typecheck_mixed.q.out
@@ -0,0 +1,99 @@
+PREHOOK: query: create table t (a string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t (a string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values ('1'),('x'),('2.0')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values ('1'),('x'),('2.0')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a SCRIPT []
+PREHOOK: query: explain
+select * from t where a in (1.0,'x',2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t where a in (1.0,'x',2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: t
+            filterExpr: ((UDFToDouble(a)) IN (1.0D, 2.0D) or (a = 'x')) (type: boolean)
+            Statistics: Num rows: 3 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: ((UDFToDouble(a)) IN (1.0D, 2.0D) or (a = 'x')) (type: boolean)
+              Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                expressions: a (type: string)
+                outputColumnNames: _col0
+                Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 2 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+      Execution mode: vectorized
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select * from t where a in (1.0,'x',2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t where a in (1.0,'x',2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+1
+x
+2.0
+PREHOOK: query: create table ax(s char(1),t char(10))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ax
+POSTHOOK: query: create table ax(s char(1),t char(10))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ax
+PREHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ax
+POSTHOOK: query: insert into ax values ('a','a'),('a','a '),('b','bb')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ax
+POSTHOOK: Lineage: ax.s SCRIPT []
+POSTHOOK: Lineage: ax.t SCRIPT []
+PREHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ax
+#### A masked pattern was here ####
+POSTHOOK: query: select 'expected 1',count(*) from ax where ((s,t) in (('a','a'),(null, 'bb'))) is null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ax
+#### A masked pattern was here ####
+expected 1	1