You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/06/05 05:38:13 UTC
svn commit: r781904 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/ppd/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: zshao
Date: Fri Jun 5 03:38:13 2009
New Revision: 781904
URL: http://svn.apache.org/viewvc?rev=781904&view=rev
Log:
HIVE-544. Fix predicate push for exprFieldNodeDesc. (Prasad Chakka via zshao)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Jun 5 03:38:13 2009
@@ -223,6 +223,9 @@
HIVE-540. Restore test.silent setting
(Zheng Shao via namit)
+ HIVE-544. Fix predicate push for exprFieldNodeDesc.
+ (Prasad Chakka via zshao)
+
Release 0.3.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java Fri Jun 5 03:38:13 2009
@@ -53,7 +53,7 @@
else
setDone(true);
}
-
+
public String getName() {
return "LIM";
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java Fri Jun 5 03:38:13 2009
@@ -141,6 +141,43 @@
}
+ public static class FieldExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ ExprWalkerInfo ctx = (ExprWalkerInfo) procCtx;
+ String alias = null;
+ exprNodeFieldDesc expr = (exprNodeFieldDesc) nd;
+
+ boolean isCandidate = true;
+ assert(nd.getChildren().size() == 1);
+ exprNodeDesc ch = (exprNodeDesc) nd.getChildren().get(0);
+ exprNodeDesc newCh = ctx.getConvertedNode(ch);
+ if (newCh != null) {
+ expr.setDesc(newCh);
+ ch = newCh;
+ }
+ String chAlias = ctx.getAlias(ch);
+
+ isCandidate = isCandidate && ctx.isCandidate(ch);
+ // need to iterate through all children even if one is found to be not a candidate
+ // in case if the other children could be individually pushed up
+ if (isCandidate && chAlias != null) {
+ if (alias == null) {
+ alias = chAlias;
+ } else if (!chAlias.equalsIgnoreCase(alias)) {
+ isCandidate = false;
+ }
+ }
+
+ ctx.addAlias(expr, alias);
+ ctx.setIsCandidate(expr, isCandidate);
+ return isCandidate;
+ }
+
+ }
+
/**
* If all children are candidates and refer only to one table alias then this expr is a candidate
* else it is not a candidate but its children could be final candidates
@@ -271,6 +308,10 @@
return new ColumnExprProcessor();
}
+ private static NodeProcessor getFieldProcessor() {
+ return new FieldExprProcessor();
+ }
+
public static ExprWalkerInfo extractPushdownPreds(OpWalkerInfo opContext,
Operator<? extends Serializable> op,
exprNodeDesc pred) throws SemanticException {
@@ -297,7 +338,7 @@
// generates the plan from the operator tree
Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
exprRules.put(new RuleRegExp("R1", exprNodeColumnDesc.class.getName() + "%"), getColumnProcessor());
- exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFuncProcessor());
+ exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFieldProcessor());
exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
exprRules.put(new RuleRegExp("R4", exprNodeIndexDesc.class.getName() + "%"), getIndexProcessor());
exprRules.put(new RuleRegExp("R5", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_testxpath4.q Fri Jun 5 03:38:13 2009
@@ -1,3 +1,20 @@
+set hive.optimize.ppd=false;
+
+EXPLAIN
+FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+ AND lintstring.myint IS NOT NULL
+ AND lintstring IS NOT NULL;
+
+FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+ OR lintstring.myint IS NOT NULL
+ OR lintstring IS NOT NULL;
+
+set hive.optimize.ppd=true;
+
EXPLAIN
FROM src_thrift
SELECT src_thrift.mstringstring['key_9'], lintstring.myint
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out?rev=781904&r1=781903&r2=781904&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out Fri Jun 5 03:38:13 2009
@@ -50,7 +50,74 @@
OR lintstring.myint IS NOT NULL
OR lintstring IS NOT NULL
Input: default/src_thrift
-Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/115422682/57144029.10000
+Output: file:/data/users/pchakka/workspace/oshive/build/ql/tmp/249993417/10000
+NULL [0]
+NULL [1]
+NULL [4]
+NULL [9]
+NULL [16]
+NULL [25]
+NULL [36]
+NULL [49]
+NULL [64]
+value_9 [81]
+query: EXPLAIN
+FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+ AND lintstring.myint IS NOT NULL
+ AND lintstring IS NOT NULL
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR ([ (. (TOK_TABLE_OR_COL src_thrift) mstringstring) 'key_9')) (TOK_SELEXPR (. (TOK_TABLE_OR_COL lintstring) myint))) (TOK_WHERE (AND (AND (TOK_FUNCTION TOK_ISNOTNULL ([ (. (TOK_TABLE_OR_COL src_thrift) mstringstring) 'key_9')) (TOK_FUNCTION TOK_ISNOTNULL (. (TOK_TABLE_OR_COL lintstring) myint))) (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL lintstring))))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src_thrift
+ Filter Operator
+ predicate:
+ expr: ((mstringstring['key_9'] is not null and lintstring.myint is not null) and lintstring is not null)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: mstringstring
+ type: map<string,string>
+ expr: lintstring
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
+ Filter Operator
+ predicate:
+ expr: ((0['key_9'] is not null and 1.myint is not null) and 1 is not null)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0['key_9']
+ type: string
+ expr: 1.myint
+ type: array<int>
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+query: FROM src_thrift
+SELECT src_thrift.mstringstring['key_9'], lintstring.myint
+WHERE src_thrift.mstringstring['key_9'] IS NOT NULL
+ OR lintstring.myint IS NOT NULL
+ OR lintstring IS NOT NULL
+Input: default/src_thrift
+Output: file:/data/users/pchakka/workspace/oshive/build/ql/tmp/1189169528/10000
NULL [0]
NULL [1]
NULL [4]