You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/25 20:42:32 UTC
[22/25] hive git commit: HIVE-11791 : Add unit test for HIVE-10122
(Illya Yalovyy via Ashutosh Chauhan, Gopal V)
HIVE-11791 : Add unit test for HIVE-10122 (Illya Yalovyy via Ashutosh Chauhan, Gopal V)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3eefcb54
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3eefcb54
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3eefcb54
Branch: refs/heads/llap
Commit: 3eefcb54f21222e5f3d3a1d097497a7e82429572
Parents: 41a12cb
Author: Illya Yalovyy <ya...@amazon.com>
Authored: Wed Sep 23 10:50:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Sep 25 07:43:23 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/ppr/PartitionPruner.java | 26 +++--
.../TestNegativePartitionPrunerCompactExpr.java | 27 +++++
.../TestPositivePartitionPrunerCompactExpr.java | 115 +++++++++++++++++++
3 files changed, 161 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3eefcb54/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
index 8eab603..5644662 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.ObjectPair;
@@ -262,7 +264,8 @@ public class PartitionPruner implements Transform {
* @param expr original partition pruning expression.
* @return partition pruning expression that only contains partition columns.
*/
- static private ExprNodeDesc compactExpr(ExprNodeDesc expr) {
+ @VisibleForTesting
+ static ExprNodeDesc compactExpr(ExprNodeDesc expr) {
// If this is a constant boolean expression, return the value.
if (expr == null) {
return null;
@@ -298,40 +301,49 @@ public class PartitionPruner implements Transform {
allTrue = false;
}
}
-
+
+ if (allTrue) {
+ return new ExprNodeConstantDesc(Boolean.TRUE);
+ }
if (newChildren.size() == 0) {
return null;
}
if (newChildren.size() == 1) {
return newChildren.get(0);
}
- if (allTrue) {
- return new ExprNodeConstantDesc(Boolean.TRUE);
- }
+
// Nothing to compact, update expr with compacted children.
((ExprNodeGenericFuncDesc) expr).setChildren(newChildren);
} else if (isOr) {
// Non-partition expressions are converted to nulls.
List<ExprNodeDesc> newChildren = new ArrayList<ExprNodeDesc>();
boolean allFalse = true;
+ boolean isNull = false;
for (ExprNodeDesc child : children) {
ExprNodeDesc compactChild = compactExpr(child);
if (compactChild != null) {
if (isTrueExpr(compactChild)) {
return new ExprNodeConstantDesc(Boolean.TRUE);
}
- if (!isFalseExpr(compactChild)) {
+ if (!isNull && !isFalseExpr(compactChild)) {
newChildren.add(compactChild);
allFalse = false;
}
} else {
- return null;
+ isNull = true;
}
}
+ if (isNull) {
+ return null;
+ }
if (allFalse) {
return new ExprNodeConstantDesc(Boolean.FALSE);
}
+ if (newChildren.size() == 1) {
+ return newChildren.get(0);
+ }
+
// Nothing to compact, update expr with compacted children.
((ExprNodeGenericFuncDesc) expr).setChildren(newChildren);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3eefcb54/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestNegativePartitionPrunerCompactExpr.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestNegativePartitionPrunerCompactExpr.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestNegativePartitionPrunerCompactExpr.java
new file mode 100644
index 0000000..36a8e63
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestNegativePartitionPrunerCompactExpr.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.junit.Test;
+
+public final class TestNegativePartitionPrunerCompactExpr {
+
+ @Test(expected = IllegalStateException.class)
+ public void testCompactExprWhenConstNonBooleanThenException() {
+ PartitionPruner.compactExpr(new ExprNodeConstantDesc("Some String"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/3eefcb54/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestPositivePartitionPrunerCompactExpr.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestPositivePartitionPrunerCompactExpr.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestPositivePartitionPrunerCompactExpr.java
new file mode 100644
index 0000000..6830ea4
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/ppr/TestPositivePartitionPrunerCompactExpr.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import java.util.Arrays;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.junit.Assert.*;
+
+@RunWith(Parameterized.class)
+public final class TestPositivePartitionPrunerCompactExpr {
+
+ private final ExprNodeDesc expression;
+ private final String expected;
+
+ public TestPositivePartitionPrunerCompactExpr(ExprNodeDesc expression, String expected) {
+ this.expression = expression;
+ this.expected = expected;
+ }
+
+ @Parameterized.Parameters(name = "{index}: {0} => {1}")
+ public static Iterable<Object[]> data() {
+ ExprNodeDesc trueExpr = new ExprNodeConstantDesc(Boolean.TRUE);
+ ExprNodeDesc falseExpr = new ExprNodeConstantDesc(Boolean.FALSE);
+ ExprNodeDesc col1Expr = new ExprNodeColumnDesc(TypeInfoFactory.booleanTypeInfo, "col1", "t1", true);
+ ExprNodeDesc col2Expr = new ExprNodeColumnDesc(TypeInfoFactory.booleanTypeInfo, "col2", "t1", true);
+ ExprNodeDesc udf1Expr = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+ new GenericUDFOPNull(), Arrays.<ExprNodeDesc>asList(col1Expr));
+ ExprNodeDesc udf2Expr = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+ new GenericUDFOPNull(), Arrays.<ExprNodeDesc>asList(col2Expr));
+
+ return Arrays.asList(new Object[][]{
+ {null, null},
+ {and(null, null), null},
+ {and(falseExpr, null), "false"},
+ {and(null, falseExpr), "false"},
+ {and(trueExpr, null), null},
+ {and(null, trueExpr), null},
+ {and(udf1Expr, null), "col1 is null"},
+ {and(null, udf2Expr), "col2 is null"},
+ {and(udf1Expr, udf2Expr), "(col1 is null and col2 is null)"},
+ {and(falseExpr, falseExpr), "false"},
+ {and(trueExpr, falseExpr), "false"},
+ {and(falseExpr, trueExpr), "false"},
+ {and(udf1Expr, falseExpr), "false"},
+ {and(falseExpr, udf2Expr), "false"},
+ {and(trueExpr, trueExpr), "true"},
+ {and(udf1Expr, trueExpr), "col1 is null"},
+ {and(trueExpr, udf2Expr), "col2 is null"},
+ {or(null, null), null},
+ {or(falseExpr, null), null},
+ {or(null, falseExpr), null},
+ {or(trueExpr, null), "true"},
+ {or(null, trueExpr), "true"},
+ {or(udf1Expr, null), null},
+ {or(null, udf2Expr), null},
+ {or(udf1Expr, udf2Expr), "(col1 is null or col2 is null)"},
+ {or(falseExpr, falseExpr), "false"},
+ {or(trueExpr, falseExpr), "true"},
+ {or(falseExpr, trueExpr), "true"},
+ {or(udf1Expr, falseExpr), "col1 is null"},
+ {or(falseExpr, udf2Expr), "col2 is null"},
+ {or(trueExpr, trueExpr), "true"},
+ {or(udf1Expr, trueExpr), "true"},
+ {or(trueExpr, udf2Expr), "true"},
+ {or(and(udf1Expr, udf2Expr), udf2Expr), "((col1 is null and col2 is null) or col2 is null)"},
+ {and(or(udf1Expr, udf2Expr), udf2Expr), "((col1 is null or col2 is null) and col2 is null)"},
+ });
+ }
+
+ @Test
+ public void testCompactExpr() {
+ ExprNodeDesc actual = PartitionPruner.compactExpr(expression);
+ if (expected == null) {
+ assertNull(actual);
+ } else {
+ assertNotNull("Expected not NULL expression", actual);
+ assertNotNull("Expected not NULL expression string", actual.getExprString());
+ assertEquals(expected, actual.getExprString());
+ }
+ }
+
+ private static ExprNodeDesc or(ExprNodeDesc left, ExprNodeDesc right) {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+ new GenericUDFOPOr(), Arrays.<ExprNodeDesc>asList(left, right));
+ }
+
+ private static ExprNodeDesc and(ExprNodeDesc left, ExprNodeDesc right) {
+ return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+ new GenericUDFOPAnd(), Arrays.<ExprNodeDesc>asList(left, right));
+ }
+}