You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/03/22 07:04:42 UTC
svn commit: r1580177 - in /hive/branches/branch-0.13/hbase-handler/src:
java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
test/queries/positive/hbase_pushdown.q
test/results/positive/hbase_pushdown.q.out
Author: hashutosh
Date: Sat Mar 22 06:04:42 2014
New Revision: 1580177
URL: http://svn.apache.org/r1580177
Log:
HIVE-6650 : hive.optimize.index.filter breaks non-index where with HBaseStorageHandler (Nick Dimiduk via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q
hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
Modified: hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1580177&r1=1580176&r2=1580177&view=diff
==============================================================================
--- hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hive/branches/branch-0.13/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Sat Mar 22 06:04:42 2014
@@ -253,6 +253,8 @@ public class HiveHBaseTableInputFormat e
private Scan createFilterScan(JobConf jobConf, int iKey, boolean isKeyBinary)
throws IOException {
+ // TODO: assert iKey is HBaseSerDe#HBASE_KEY_COL
+
Scan scan = new Scan();
String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
if (filterExprSerialized == null) {
@@ -270,18 +272,15 @@ public class HiveHBaseTableInputFormat e
ExprNodeDesc residualPredicate =
analyzer.analyzePredicate(filterExpr, searchConditions);
- // There should be no residual since we already negotiated
- // that earlier in HBaseStorageHandler.decomposePredicate.
+ // There should be no residual since we already negotiated that earlier in
+ // HBaseStorageHandler.decomposePredicate. However, with hive.optimize.index.filter
+ // OpProcFactory#pushFilterToStorageHandler pushes the original filter back down again.
+ // Since pushed-down filters are not ommitted at the higher levels (and thus the
+ // contract of negotiation is ignored anyway), just ignore the residuals.
+ // Re-assess this when negotiation is honored and the duplicate evaluation is removed.
+ // THIS IGNORES RESIDUAL PARSING FROM HBaseStorageHandler#decomposePredicate
if (residualPredicate != null) {
- throw new RuntimeException(
- "Unexpected residual predicate " + residualPredicate.getExprString());
- }
-
- // There should be exactly one predicate since we already
- // negotiated that also.
- if (searchConditions.size() < 1 || searchConditions.size() > 2) {
- throw new RuntimeException(
- "Either one or two search conditions expected in push down");
+ LOG.debug("Ignoring residual predicate " + residualPredicate.getExprString());
}
// Convert the search condition into a restriction on the HBase scan
@@ -292,7 +291,7 @@ public class HiveHBaseTableInputFormat e
PrimitiveObjectInspector objInspector;
Object writable;
- try{
+ try {
objInspector = (PrimitiveObjectInspector)eval.initialize(null);
writable = eval.evaluate(null);
} catch (ClassCastException cce) {
Modified: hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q?rev=1580177&r1=1580176&r2=1580177&view=diff
==============================================================================
--- hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q (original)
+++ hive/branches/branch-0.13/hbase-handler/src/test/queries/positive/hbase_pushdown.q Sat Mar 22 06:04:42 2014
@@ -17,6 +17,12 @@ explain select * from hbase_pushdown whe
select * from hbase_pushdown where key=90 and value like '%90%';
+set hive.optimize.index.filter=true;
+-- with partial pushdown with optimization (HIVE-6650)
+explain select * from hbase_pushdown where key=90 and value like '%90%';
+select * from hbase_pushdown where key=90 and value like '%90%';
+set hive.optimize.index.filter=false;
+
-- with two residuals
explain select * from hbase_pushdown
Modified: hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out?rev=1580177&r1=1580176&r2=1580177&view=diff
==============================================================================
--- hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out (original)
+++ hive/branches/branch-0.13/hbase-handler/src/test/results/positive/hbase_pushdown.q.out Sat Mar 22 06:04:42 2014
@@ -115,6 +115,52 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_pushdown
#### A masked pattern was here ####
90 val_90
+PREHOOK: query: -- with partial pushdown with optimization (HIVE-6650)
+explain select * from hbase_pushdown where key=90 and value like '%90%'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- with partial pushdown with optimization (HIVE-6650)
+explain select * from hbase_pushdown where key=90 and value like '%90%'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: hbase_pushdown
+ filterExpr: (key = 90) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Filter Operator
+ predicate: (value like '%90%') (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: select * from hbase_pushdown where key=90 and value like '%90%'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_pushdown
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_pushdown where key=90 and value like '%90%'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_pushdown
+#### A masked pattern was here ####
+90 val_90
PREHOOK: query: -- with two residuals
explain select * from hbase_pushdown