You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2008/12/31 04:30:54 UTC
svn commit: r730302 - in /hadoop/hive/trunk: CHANGES.txt
ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
ql/src/test/queries/clientpositive/join19.q
ql/src/test/results/clientpositive/join19.q.out
Author: zshao
Date: Tue Dec 30 19:30:54 2008
New Revision: 730302
URL: http://svn.apache.org/viewvc?rev=730302&view=rev
Log:
HIVE-151. Tasks depending on other tasks cannot be a root task. (Namit Jain via zshao)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join19.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=730302&r1=730301&r2=730302&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Dec 30 19:30:54 2008
@@ -69,6 +69,9 @@
BUG FIXES
+ HIVE-151. Tasks depending on other tasks cannot be a root task.
+ (Namit Jain via zshao)
+
HIVE-197. Fixed UDFToBoolean. (zshao)
HIVE-171. Fixed select star with cluster by, distribute by and sort by.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=730302&r1=730301&r2=730302&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Tue Dec 30 19:30:54 2008
@@ -298,16 +298,20 @@
* @param opProcCtx context
**/
private static void splitTasks(ReduceSinkOperator op,
- Task<? extends Serializable> oldTask,
- Task<? extends Serializable> task,
+ Task<? extends Serializable> parentTask,
+ Task<? extends Serializable> childTask,
GenMRProcContext opProcCtx) throws SemanticException {
- Task<? extends Serializable> currTask = task;
- mapredWork plan = (mapredWork) currTask.getWork();
+ mapredWork plan = (mapredWork) childTask.getWork();
Operator<? extends Serializable> currTopOp = opProcCtx.getCurrTopOp();
ParseContext parseCtx = opProcCtx.getParseCtx();
- oldTask.addDependentTask(currTask);
-
+ parentTask.addDependentTask(childTask);
+
+ // Root Task cannot depend on any other task, therefore childTask cannot be a root Task
+ List<Task<? extends Serializable>> rootTasks = opProcCtx.getRootTasks();
+ if (rootTasks.contains(childTask))
+ rootTasks.remove(childTask);
+
// generate the temporary file
String scratchDir = opProcCtx.getScratchDir();
int randomid = opProcCtx.getRandomId();
@@ -344,7 +348,7 @@
Operator<? extends Serializable> reducer = op.getChildOperators().get(0);
String streamDesc;
- mapredWork cplan = (mapredWork) currTask.getWork();
+ mapredWork cplan = (mapredWork) childTask.getWork();
if (reducer.getClass() == JoinOperator.class) {
String origStreamDesc;
@@ -377,6 +381,6 @@
opProcCtx.setCurrTopOp(currTopOp);
opProcCtx.setCurrAliasId(currAliasId);
- opProcCtx.setCurrTask(currTask);
+ opProcCtx.setCurrTask(childTask);
}
}
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join19.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join19.q?rev=730302&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join19.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join19.q Tue Dec 30 19:30:54 2008
@@ -0,0 +1,58 @@
+drop TABLE triples;
+CREATE TABLE triples (foo string, subject string, predicate string, object string, foo2 string);
+
+EXPLAIN
+SELECT t11.subject, t22.object , t33.subject , t55.object, t66.object
+FROM
+(
+SELECT t1.subject
+FROM triples t1
+WHERE
+t1.predicate='http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL'
+AND
+t1.object='http://ontos/OntosMiner/Common.English/ontology#Citation'
+) t11
+JOIN
+(
+SELECT t2.subject , t2.object
+FROM triples t2
+WHERE
+t2.predicate='http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL'
+) t22
+ON (t11.subject=t22.subject)
+JOIN
+(
+SELECT t3.subject , t3.object
+FROM triples t3
+WHERE
+t3.predicate='http://www.ontosearch.com/2007/12/ontosofa-ns#_from'
+
+) t33
+ON (t11.subject=t33.object)
+JOIN
+(
+SELECT t4.subject
+FROM triples t4
+WHERE
+t4.predicate='http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL'
+AND
+t4.object='http://ontos/OntosMiner/Common.English/ontology#Author'
+
+) t44
+ON (t44.subject=t33.subject)
+JOIN
+(
+SELECT t5.subject, t5.object
+FROM triples t5
+WHERE
+t5.predicate='http://www.ontosearch.com/2007/12/ontosofa-ns#_to'
+) t55
+ON (t55.subject=t44.subject)
+JOIN
+(
+SELECT t6.subject, t6.object
+FROM triples t6
+WHERE
+t6.predicate='http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL'
+) t66
+ON (t66.subject=t55.object);
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out?rev=730302&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join19.q.out Tue Dec 30 19:30:54 2008
@@ -0,0 +1,342 @@
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_JOIN (TOK_JOIN (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF triples t1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t1 subject))) (TOK_WHERE (AND (= (TOK_COLREF t1 predicate) 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') (= (TOK_COLREF t1 object) 'http://ontos/OntosMiner/Common.English/ontology#Citation'))))) t11) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF triples t2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t2 subject)) (TOK_SELEXPR (TOK_COLREF t2 object))) (TOK_WHERE (= (TOK_COLREF t2 predicate) 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL')))) t22) (= (TOK_COLREF t11 subject) (TOK_COLREF t22 subject))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF triples t3)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t3 subject)) (TOK_SELEXPR (TOK_CO
LREF t3 object))) (TOK_WHERE (= (TOK_COLREF t3 predicate) 'http://www.ontosearch.com/2007/12/ontosofa-ns#_from')))) t33) (= (TOK_COLREF t11 subject) (TOK_COLREF t33 object))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF triples t4)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t4 subject))) (TOK_WHERE (AND (= (TOK_COLREF t4 predicate) 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') (= (TOK_COLREF t4 object) 'http://ontos/OntosMiner/Common.English/ontology#Author'))))) t44) (= (TOK_COLREF t44 subject) (TOK_COLREF t33 subject))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF triples t5)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t5 subject)) (TOK_SELEXPR (TOK_COLREF t5 object))) (TOK_WHERE (= (TOK_COLREF t5 predicate) 'http://www.ontosearch.com/2007/12/ontosofa-ns#_to')))) t55) (= (TOK_COLREF t55 subject) (TOK_COLREF t44 subject))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM
(TOK_TABREF triples t6)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t6 subject)) (TOK_SELEXPR (TOK_COLREF t6 object))) (TOK_WHERE (= (TOK_COLREF t6 predicate) 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL')))) t66) (= (TOK_COLREF t66 subject) (TOK_COLREF t55 object)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF t11 subject)) (TOK_SELEXPR (TOK_COLREF t22 object)) (TOK_SELEXPR (TOK_COLREF t33 subject)) (TOK_SELEXPR (TOK_COLREF t55 object)) (TOK_SELEXPR (TOK_COLREF t66 object)))))
+
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-3 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-3
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ t22:t2
+ Select Operator
+ expressions:
+ expr: subject
+ type: string
+ expr: object
+ type: string
+ expr: predicate
+ type: string
+ Filter Operator
+ predicate:
+ expr: (2 = 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: string
+ tag: 1
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ t33:t3
+ Select Operator
+ expressions:
+ expr: subject
+ type: string
+ expr: object
+ type: string
+ expr: predicate
+ type: string
+ Filter Operator
+ predicate:
+ expr: (2 = 'http://www.ontosearch.com/2007/12/ontosofa-ns#_from')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 1
+ type: string
+ tag: 2
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ t11:t1
+ Select Operator
+ expressions:
+ expr: subject
+ type: string
+ expr: predicate
+ type: string
+ expr: object
+ type: string
+ Filter Operator
+ predicate:
+ expr: ((1 = 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') and (2 = 'http://ontos/OntosMiner/Common.English/ontology#Citation'))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: string
+ tag: 0
+ value expressions:
+ expr: 0
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE.0}
+ 1 {VALUE.0} {VALUE.1}
+ 2 {VALUE.0} {VALUE.1}
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+ name: binary_table
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: 3
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 3
+ type: string
+ tag: 0
+ value expressions:
+ expr: 3
+ type: string
+ expr: 4
+ type: string
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ expr: 2
+ type: string
+ t55:t5
+ Select Operator
+ expressions:
+ expr: subject
+ type: string
+ expr: object
+ type: string
+ expr: predicate
+ type: string
+ Filter Operator
+ predicate:
+ expr: (2 = 'http://www.ontosearch.com/2007/12/ontosofa-ns#_to')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: string
+ tag: 2
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ t44:t4
+ Select Operator
+ expressions:
+ expr: subject
+ type: string
+ expr: predicate
+ type: string
+ expr: object
+ type: string
+ Filter Operator
+ predicate:
+ expr: ((1 = 'http://sofa.semanticweb.org/sofa/v1.0/system#__INSTANCEOF_REL') and (2 = 'http://ontos/OntosMiner/Common.English/ontology#Author'))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: string
+ tag: 1
+ value expressions:
+ expr: 0
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 1 to 1
+ condition expressions:
+ 0 {VALUE.0} {VALUE.1} {VALUE.2} {VALUE.3} {VALUE.4}
+ 1 {VALUE.0}
+ 2 {VALUE.0} {VALUE.1}
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
+ name: binary_table
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ t66:t6
+ Select Operator
+ expressions:
+ expr: subject
+ type: string
+ expr: object
+ type: string
+ expr: predicate
+ type: string
+ Filter Operator
+ predicate:
+ expr: (2 = 'http://sofa.semanticweb.org/sofa/v1.0/system#__LABEL_REL')
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: string
+ tag: 1
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: 7
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: 7
+ type: string
+ tag: 0
+ value expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ expr: 5
+ type: string
+ expr: 2
+ type: string
+ expr: 3
+ type: string
+ expr: 4
+ type: string
+ expr: 6
+ type: string
+ expr: 7
+ type: string
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE.0} {VALUE.1} {VALUE.2} {VALUE.3} {VALUE.4} {VALUE.5} {VALUE.6} {VALUE.7}
+ 1 {VALUE.0} {VALUE.1}
+ Select Operator
+ expressions:
+ expr: 3
+ type: string
+ expr: 5
+ type: string
+ expr: 0
+ type: string
+ expr: 7
+ type: string
+ expr: 9
+ type: string
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE.0} {VALUE.1} {VALUE.2} {VALUE.3} {VALUE.4} {VALUE.5} {VALUE.6} {VALUE.7}
+ 1 {VALUE.0} {VALUE.1}
+ Select Operator
+ expressions:
+ expr: 3
+ type: string
+ expr: 5
+ type: string
+ expr: 0
+ type: string
+ expr: 7
+ type: string
+ expr: 9
+ type: string
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+