You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/04/03 11:04:43 UTC
svn commit: r1308763 - in /hive/trunk:
hbase-handler/src/test/queries/hbase_ppd_key_range.q
hbase-handler/src/test/results/hbase_ppd_key_range.q.out
ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
Author: hashutosh
Date: Tue Apr 3 09:04:42 2012
New Revision: 1308763
URL: http://svn.apache.org/viewvc?rev=1308763&view=rev
Log:
HIVE-2861 : Support eventual constant expression for filter pushdown for key ranges in hbase (navis via Ashutosh Chauhan)
Modified:
hive/trunk/hbase-handler/src/test/queries/hbase_ppd_key_range.q
hive/trunk/hbase-handler/src/test/results/hbase_ppd_key_range.q.out
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
Modified: hive/trunk/hbase-handler/src/test/queries/hbase_ppd_key_range.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/hbase_ppd_key_range.q?rev=1308763&r1=1308762&r2=1308763&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/hbase_ppd_key_range.q (original)
+++ hive/trunk/hbase-handler/src/test/queries/hbase_ppd_key_range.q Tue Apr 3 09:04:42 2012
@@ -14,6 +14,10 @@ select * from hbase_pushdown where key<'
select * from hbase_pushdown where key<='2';
select * from hbase_pushdown where key>='90';
+-- with cnostant expressinon
+explain select * from hbase_pushdown where key>=cast(40 + 50 as string);
+select * from hbase_pushdown where key>=cast(40 + 50 as string);
+
-- with partial pushdown
explain select * from hbase_pushdown where key>'90' and value like '%9%';
Modified: hive/trunk/hbase-handler/src/test/results/hbase_ppd_key_range.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/hbase_ppd_key_range.q.out?rev=1308763&r1=1308762&r2=1308763&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/hbase_ppd_key_range.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/hbase_ppd_key_range.q.out Tue Apr 3 09:04:42 2012
@@ -182,6 +182,66 @@ POSTHOOK: Input: default@hbase_pushdown
96 val_96
97 val_97
98 val_98
+PREHOOK: query: -- with cnostant expressinon
+explain select * from hbase_pushdown where key>=cast(40 + 50 as string)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- with cnostant expressinon
+explain select * from hbase_pushdown where key>=cast(40 + 50 as string)
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME hbase_pushdown))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (>= (TOK_TABLE_OR_COL key) (TOK_FUNCTION TOK_STRING (+ 40 50))))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ hbase_pushdown
+ TableScan
+ alias: hbase_pushdown
+ filterExpr:
+ expr: (key >= UDFToString((40 + 50)))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key >= UDFToString((40 + 50)))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select * from hbase_pushdown where key>=cast(40 + 50 as string)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_pushdown
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_pushdown where key>=cast(40 + 50 as string)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_pushdown
+#### A masked pattern was here ####
+90 val_90
+92 val_92
+95 val_95
+96 val_96
+97 val_97
+98 val_98
PREHOOK: query: -- with partial pushdown
explain select * from hbase_pushdown where key>'90' and value like '%9%'
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java?rev=1308763&r1=1308762&r2=1308763&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java Tue Apr 3 09:04:42 2012
@@ -26,7 +26,10 @@ import java.util.Map;
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -40,10 +43,14 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ReflectionUtils;
/**
* IndexPredicateAnalyzer decomposes predicates, separating the parts
@@ -58,7 +65,7 @@ public class IndexPredicateAnalyzer
private Set<String> udfNames;
private Set<String> allowedColumnNames;
-
+
public IndexPredicateAnalyzer() {
udfNames = new HashSet<String>();
}
@@ -186,8 +193,8 @@ public class IndexPredicateAnalyzer
return expr;
}
- ExprNodeDesc child1 = (ExprNodeDesc) nodeOutputs[0];
- ExprNodeDesc child2 = (ExprNodeDesc) nodeOutputs[1];
+ ExprNodeDesc child1 = extractConstant((ExprNodeDesc) nodeOutputs[0]);
+ ExprNodeDesc child2 = extractConstant((ExprNodeDesc) nodeOutputs[1]);
ExprNodeColumnDesc columnDesc = null;
ExprNodeConstantDesc constantDesc = null;
if ((child1 instanceof ExprNodeColumnDesc)
@@ -221,6 +228,55 @@ public class IndexPredicateAnalyzer
return null;
}
+ private ExprNodeDesc extractConstant(ExprNodeDesc expr) {
+ if (!(expr instanceof ExprNodeGenericFuncDesc)) {
+ return expr;
+ }
+ ExprNodeConstantDesc folded = foldConstant(((ExprNodeGenericFuncDesc) expr));
+ return folded == null ? expr : folded;
+ }
+
+ private ExprNodeConstantDesc foldConstant(ExprNodeGenericFuncDesc func) {
+ GenericUDF udf = func.getGenericUDF();
+ if (!FunctionRegistry.isDeterministic(udf) || FunctionRegistry.isStateful(udf)) {
+ return null;
+ }
+ try {
+ // If the UDF depends on any external resources, we can't fold because the
+ // resources may not be available at compile time.
+ if (udf instanceof GenericUDFBridge) {
+ UDF internal = ReflectionUtils.newInstance(((GenericUDFBridge) udf).getUdfClass(), null);
+ if (internal.getRequiredFiles() != null || internal.getRequiredJars() != null) {
+ return null;
+ }
+ } else {
+ if (udf.getRequiredFiles() != null || udf.getRequiredJars() != null) {
+ return null;
+ }
+ }
+
+ for (ExprNodeDesc child : func.getChildExprs()) {
+ if (child instanceof ExprNodeConstantDesc) {
+ continue;
+ } else if (child instanceof ExprNodeGenericFuncDesc) {
+ if (foldConstant((ExprNodeGenericFuncDesc) child) != null) {
+ continue;
+ }
+ }
+ return null;
+ }
+ ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(func);
+ ObjectInspector output = evaluator.initialize(null);
+
+ Object constant = evaluator.evaluate(null);
+ Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
+
+ return new ExprNodeConstantDesc(java);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
/**
* Translates search conditions back to ExprNodeDesc form (as
* a left-deep conjunction).