You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/09/05 22:46:43 UTC
hive git commit: HIVE-18873: Skipping predicate pushdown for MR
silently at HiveInputFormat can cause storage handlers to produce erroneous
result(Ankit Singhal, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 56d3cd42a -> 6baeac7d6
HIVE-18873: Skipping predicate pushdown for MR silently at HiveInputFormat can cause storage handlers to produce erroneous result(Ankit Singhal, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6baeac7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6baeac7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6baeac7d
Branch: refs/heads/master
Commit: 6baeac7d6641bdf92a3ad877f825f4989d03ebfd
Parents: 56d3cd4
Author: Vineet Garg <vg...@apache.org>
Authored: Wed Sep 5 15:45:04 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Wed Sep 5 15:45:04 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/cli/control/CliConfigs.java | 1 +
.../java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java | 9 ++++++---
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6baeac7d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 491b6db..3466bac 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -520,6 +520,7 @@ public class CliConfigs {
super(CoreAccumuloCliDriver.class);
try {
setQueryDir("accumulo-handler/src/test/queries/positive");
+ excludeQuery("accumulo_joins.q");
setResultsDir("accumulo-handler/src/test/results/positive");
setLogDir("itests/qtest/target/qfile-results/accumulo-handler/positive");
http://git-wip-us.apache.org/repos/asf/hive/blob/6baeac7d/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index ea37daf..a86656c 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkDynamicPartitionPruner;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -822,10 +823,12 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
return;
}
- // disable filter pushdown for mapreduce when there are more than one table aliases,
+ // disable filter pushdown for mapreduce(except for storage handlers) when there are more than one table aliases,
// since we don't clone jobConf per alias
- if (mrwork != null && mrwork.getAliases() != null && mrwork.getAliases().size() > 1 &&
- jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname).equals("mr")) {
+ if (mrwork != null && mrwork.getAliases() != null && mrwork.getAliases().size() > 1
+ && jobConf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname).equals("mr")
+ && (scanDesc.getTableMetadata() == null
+ || !(scanDesc.getTableMetadata().getStorageHandler() instanceof HiveStoragePredicateHandler))) {
return;
}