You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2013/04/16 04:35:09 UTC
svn commit: r1468292 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/ test/results/clientpositive/
Author: navis
Date: Tue Apr 16 02:35:08 2013
New Revision: 1468292
URL: http://svn.apache.org/r1468292
Log:
HIVE-4261 union_remove_10 is failing on hadoop2 with assertion (root task with non-empty set of parents)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=1468292&r1=1468291&r2=1468292&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Tue Apr 16 02:35:08 2013
@@ -204,7 +204,9 @@ public class GenMRFileSink1 implements N
(MapredWork) currTask.getWork(), false, ctx);
}
- if (!rootTasks.contains(currTask)) {
+ if (!rootTasks.contains(currTask)
+ && (currTask.getParentTasks() == null
+ || currTask.getParentTasks().isEmpty())) {
rootTasks.add(currTask);
}
}
@@ -721,7 +723,9 @@ public class GenMRFileSink1 implements N
(MapredWork) currTask.getWork(), false, ctx);
}
opTaskMap.put(null, currTask);
- if (!rootTasks.contains(currTask)) {
+ if (!rootTasks.contains(currTask)
+ && (currTask.getParentTasks() == null
+ || currTask.getParentTasks().isEmpty())) {
rootTasks.add(currTask);
}
} else {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java?rev=1468292&r1=1468291&r2=1468292&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRUnion1.java Tue Apr 16 02:35:08 2013
@@ -264,7 +264,9 @@ public class GenMRUnion1 implements Node
else {
// is the current task a root task
if (shouldBeRootTask(currTask)
- && (!ctx.getRootTasks().contains(currTask))) {
+ && !ctx.getRootTasks().contains(currTask)
+ && (currTask.getParentTasks() == null
+ || currTask.getParentTasks().isEmpty())) {
ctx.getRootTasks().add(currTask);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1468292&r1=1468291&r2=1468292&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Tue Apr 16 02:35:08 2013
@@ -106,7 +106,9 @@ public final class GenMapRedUtils {
List<Task<? extends Serializable>> rootTasks = opProcCtx.getRootTasks();
- if (!rootTasks.contains(currTask)) {
+ if (!rootTasks.contains(currTask)
+ && (currTask.getParentTasks() == null
+ || currTask.getParentTasks().isEmpty())) {
rootTasks.add(currTask);
}
if (reducer.getClass() == JoinOperator.class) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java?rev=1468292&r1=1468291&r2=1468292&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java Tue Apr 16 02:35:08 2013
@@ -163,8 +163,11 @@ public final class MapJoinFactory {
opTaskMap.put(op, currTask);
List<Task<? extends Serializable>> rootTasks = opProcCtx.getRootTasks();
- assert (!rootTasks.contains(currTask));
- rootTasks.add(currTask);
+ if(!rootTasks.contains(currTask)
+ && (currTask.getParentTasks() == null
+ || currTask.getParentTasks().isEmpty())) {
+ rootTasks.add(currTask);
+ }
assert currTopOp != null;
opProcCtx.getSeenOps().add(currTopOp);
Modified: hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out?rev=1468292&r1=1468291&r2=1468292&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union_remove_10.q.out Tue Apr 16 02:35:08 2013
@@ -152,6 +152,25 @@ STAGE PLANS:
expr: _col1
type: bigint
outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
+ null-subquery2:b-subquery2-subquery2:a-subquery2:inputtbl1
+ TableScan
+ alias: inputtbl1
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: UDFToLong(2)
+ type: bigint
+ outputColumnNames: _col0, _col1
+ Union
Select Operator
expressions:
expr: _col0
@@ -167,46 +186,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
- null-subquery2:b-subquery2-subquery2:a-subquery2:inputtbl1
- TableScan
- alias: inputtbl1
- Select Operator
- expressions:
- expr: key
- type: string
- expr: 2
- type: int
- outputColumnNames: _col0, _col1
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: UDFToLong(_col1)
- type: bigint
- outputColumnNames: _col0, _col1
- Union
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: bigint
- outputColumnNames: _col0, _col1
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: bigint
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 1
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
Stage: Stage-7
Conditional Operator
@@ -249,31 +228,17 @@ STAGE PLANS:
expressions:
expr: key
type: string
- expr: 1
- type: int
+ expr: UDFToLong(1)
+ type: bigint
outputColumnNames: _col0, _col1
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: UDFToLong(_col1)
- type: bigint
- outputColumnNames: _col0, _col1
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: bigint
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 1
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
PREHOOK: query: insert overwrite table outputTbl1