You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/05/10 23:53:37 UTC
hive git commit: HIVE-16609: col='__HIVE_DEFAULT_PARTITION__'
condition in select statement may produce wrong result (Daniel Dai,
reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 2e0d02b7d -> c550b7de6
HIVE-16609: col='__HIVE_DEFAULT_PARTITION__' condition in select statement may produce wrong result (Daniel Dai, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c550b7de
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c550b7de
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c550b7de
Branch: refs/heads/master
Commit: c550b7de6514ba7dbf91402cd3b3f33a3cc8d631
Parents: 2e0d02b
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed May 10 16:53:17 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Wed May 10 16:53:17 2017 -0700
----------------------------------------------------------------------
.../ppr/PartitionExpressionForMetastore.java | 8 +++
.../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 53 ++++++++++++++++++++
.../clientpositive/alter_partition_change_col.q | 1 +
.../alter_partition_change_col.q.out | 20 ++++++++
4 files changed, 82 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c550b7de/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
index 96910e3..6555187 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.slf4j.Logger;
@@ -53,6 +55,12 @@ public class PartitionExpressionForMetastore implements PartitionExpressionProxy
String defaultPartitionName, List<String> partitionNames) throws MetaException {
ExprNodeGenericFuncDesc expr = deserializeExpr(exprBytes);
try {
+ ExprNodeDescUtils.replaceEqualDefaultPartition(expr, defaultPartitionName);
+ } catch (SemanticException ex) {
+ LOG.error("Failed to replace default partition", ex);
+ throw new MetaException(ex.getMessage());
+ }
+ try {
long startTime = System.nanoTime(), len = partitionNames.size();
boolean result = PartitionPruner.prunePartitionNames(
partColumnNames, partColumnTypeInfos, expr, defaultPartitionName, partitionNames);
http://git-wip-us.apache.org/repos/asf/hive/blob/c550b7de/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
index 01fab9c..8701b2d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
@@ -43,6 +45,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.common.collect.Lists;
+
public class ExprNodeDescUtils {
public static int indexOf(ExprNodeDesc origin, List<ExprNodeDesc> sources) {
@@ -91,6 +95,55 @@ public class ExprNodeDescUtils {
return origin;
}
+ private static boolean isDefaultPartition(ExprNodeDesc origin, String defaultPartitionName) {
+ if (origin instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc)origin).getValue() != null &&
+ ((ExprNodeConstantDesc)origin).getValue().equals(defaultPartitionName)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public static void replaceEqualDefaultPartition(ExprNodeDesc origin,
+ String defaultPartitionName) throws SemanticException {
+ ExprNodeColumnDesc column = null;
+ ExprNodeConstantDesc defaultPartition = null;
+ if (origin instanceof ExprNodeGenericFuncDesc
+ && (((ExprNodeGenericFuncDesc) origin)
+ .getGenericUDF() instanceof GenericUDFOPEqual
+ || ((ExprNodeGenericFuncDesc) origin)
+ .getGenericUDF() instanceof GenericUDFOPNotEqual)) {
+ if (isDefaultPartition(origin.getChildren().get(0),
+ defaultPartitionName)) {
+ defaultPartition = (ExprNodeConstantDesc) origin.getChildren().get(0);
+ column = (ExprNodeColumnDesc) origin.getChildren().get(1);
+ } else if (isDefaultPartition(origin.getChildren().get(1),
+ defaultPartitionName)) {
+ column = (ExprNodeColumnDesc) origin.getChildren().get(0);
+ defaultPartition = (ExprNodeConstantDesc) origin.getChildren().get(1);
+ }
+ }
+ // Found
+ if (column != null) {
+ origin.getChildren().remove(defaultPartition);
+ String fnName;
+ if (((ExprNodeGenericFuncDesc) origin)
+ .getGenericUDF() instanceof GenericUDFOPEqual) {
+ fnName = "isnull";
+ } else {
+ fnName = "isnotnull";
+ }
+ ((ExprNodeGenericFuncDesc) origin).setGenericUDF(
+ FunctionRegistry.getFunctionInfo(fnName).getGenericUDF());
+ } else {
+ if (origin.getChildren() != null) {
+ for (ExprNodeDesc child : origin.getChildren()) {
+ replaceEqualDefaultPartition(child, defaultPartitionName);
+ }
+ }
+ }
+ }
+
/**
* return true if predicate is already included in source
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/c550b7de/ql/src/test/queries/clientpositive/alter_partition_change_col.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alter_partition_change_col.q b/ql/src/test/queries/clientpositive/alter_partition_change_col.q
index 23de3d7..fe95176 100644
--- a/ql/src/test/queries/clientpositive/alter_partition_change_col.q
+++ b/ql/src/test/queries/clientpositive/alter_partition_change_col.q
@@ -17,6 +17,7 @@ insert overwrite table alter_partition_change_col1 partition (p1, p2)
show partitions alter_partition_change_col1;
select * from alter_partition_change_col1 where p1='abc';
select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__';
+select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' or lower(p1)='a';
-- Change c2 to decimal(10,0)
alter table alter_partition_change_col1 change c2 c2 decimal(10,0);
http://git-wip-us.apache.org/repos/asf/hive/blob/c550b7de/ql/src/test/results/clientpositive/alter_partition_change_col.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_change_col.q.out b/ql/src/test/results/clientpositive/alter_partition_change_col.q.out
index 09dfd1c..1738aea 100644
--- a/ql/src/test/results/clientpositive/alter_partition_change_col.q.out
+++ b/ql/src/test/results/clientpositive/alter_partition_change_col.q.out
@@ -89,6 +89,26 @@ Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123
Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123
Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123
Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123
+PREHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' or lower(p1)='a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alter_partition_change_col1
+PREHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123
+#### A masked pattern was here ####
+POSTHOOK: query: select * from alter_partition_change_col1 where p1='__HIVE_DEFAULT_PARTITION__' or lower(p1)='a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alter_partition_change_col1
+POSTHOOK: Input: default@alter_partition_change_col1@p1=__HIVE_DEFAULT_PARTITION__/p2=123
+#### A masked pattern was here ####
+Beck 0.0 __HIVE_DEFAULT_PARTITION__ 123
+Beck 77.341 __HIVE_DEFAULT_PARTITION__ 123
+Beck 79.9 __HIVE_DEFAULT_PARTITION__ 123
+Cluck 5.96 __HIVE_DEFAULT_PARTITION__ 123
+Mary 33.33 __HIVE_DEFAULT_PARTITION__ 123
+Mary 4.329 __HIVE_DEFAULT_PARTITION__ 123
+Snow 55.71 __HIVE_DEFAULT_PARTITION__ 123
+Tom -12.25 __HIVE_DEFAULT_PARTITION__ 123
+Tom 19.00 __HIVE_DEFAULT_PARTITION__ 123
+Tom 234.79 __HIVE_DEFAULT_PARTITION__ 123
PREHOOK: query: alter table alter_partition_change_col1 change c2 c2 decimal(10,0)
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@alter_partition_change_col1