You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/09/05 22:46:43 UTC

hive git commit: HIVE-18873: Skipping predicate pushdown for MR silently at HiveInputFormat can cause storage handlers to produce erroneous result(Ankit Singhal, reviewed by Prasanth Jayachandran)

Repository: hive
Updated Branches:
  refs/heads/master 56d3cd42a -> 6baeac7d6


HIVE-18873: Skipping predicate pushdown for MR silently at HiveInputFormat can cause storage handlers to produce erroneous result(Ankit Singhal, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6baeac7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6baeac7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6baeac7d

Branch: refs/heads/master
Commit: 6baeac7d6641bdf92a3ad877f825f4989d03ebfd
Parents: 56d3cd4
Author: Vineet Garg <vg...@apache.org>
Authored: Wed Sep 5 15:45:04 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Wed Sep 5 15:45:04 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/cli/control/CliConfigs.java | 1 +
 .../java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java  | 9 ++++++---
 2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6baeac7d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 491b6db..3466bac 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -520,6 +520,7 @@ public class CliConfigs {
       super(CoreAccumuloCliDriver.class);
       try {
         setQueryDir("accumulo-handler/src/test/queries/positive");
+        excludeQuery("accumulo_joins.q");
 
         setResultsDir("accumulo-handler/src/test/results/positive");
         setLogDir("itests/qtest/target/qfile-results/accumulo-handler/positive");

http://git-wip-us.apache.org/repos/asf/hive/blob/6baeac7d/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index ea37daf..a86656c 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkDynamicPartitionPruner;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -822,10 +823,12 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
       return;
     }
 
-    // disable filter pushdown for mapreduce when there are more than one table aliases,
+    // disable filter pushdown for mapreduce(except for storage handlers) when there are more than one table aliases,
     // since we don't clone jobConf per alias
-    if (mrwork != null && mrwork.getAliases() != null && mrwork.getAliases().size() > 1 &&
-      jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname).equals("mr")) {
+    if (mrwork != null && mrwork.getAliases() != null && mrwork.getAliases().size() > 1
+        && jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname).equals("mr")
+        && (scanDesc.getTableMetadata() == null
+            || !(scanDesc.getTableMetadata().getStorageHandler() instanceof HiveStoragePredicateHandler))) {
       return;
     }