You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/11/27 07:05:19 UTC
svn commit: r1642026 - in /hive/branches/spark: itests/src/test/resources/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/
ql/src/test/results/clientpositive/spark/
Author: szehon
Date: Thu Nov 27 06:05:19 2014
New Revision: 1642026
URL: http://svn.apache.org/r1642026
Log:
HIVE-8924 : Investigate test failure for join_empty.q [Spark Branch] (Szehon, reviewed by Xuefu Zhang)
Modified:
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Thu Nov 27 06:05:19 2014
@@ -657,6 +657,7 @@ spark.query.files=add_part_multiple.q, \
join_cond_pushdown_unqual2.q, \
join_cond_pushdown_unqual3.q, \
join_cond_pushdown_unqual4.q, \
+ join_empty.q \
join_filters.q, \
join_filters_overlap.q, \
join_hive_626.q, \
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SparkMapJoinResolver.java Thu Nov 27 06:05:19 2014
@@ -21,7 +21,9 @@ package org.apache.hadoop.hive.ql.optimi
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -42,6 +44,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SparkWork;
@@ -63,7 +66,24 @@ public class SparkMapJoinResolver implem
// Check whether the specified BaseWork's operator tree contains a operator
// of the specified operator class
private boolean containsOp(BaseWork work, Class<?> clazz) {
- for (Operator<? extends OperatorDesc> op : work.getAllOperators()) {
+ Set<Operator<? extends OperatorDesc>> ops = new HashSet<Operator<? extends OperatorDesc>>();
+ if (work instanceof MapWork) {
+ Collection<Operator<?>> opSet = ((MapWork) work).getAliasToWork().values();
+ Stack<Operator<?>> opStack = new Stack<Operator<?>>();
+ opStack.addAll(opSet);
+
+ while (!opStack.empty()) {
+ Operator<?> op = opStack.pop();
+ ops.add(op);
+ if (op.getChildOperators() != null) {
+ opStack.addAll(op.getChildOperators());
+ }
+ }
+ } else {
+ ops.addAll(work.getAllOperators());
+ }
+
+ for (Operator<? extends OperatorDesc> op : ops) {
if (clazz.isInstance(op)) {
return true;
}
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out Thu Nov 27 06:05:19 2014 differ
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out Thu Nov 27 06:05:19 2014 differ
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out Thu Nov 27 06:05:19 2014
@@ -43,18 +43,26 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select * from v where ds='2011-09-01'
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Map 2 <- Map 1 (NONE, 0)
#### A masked pattern was here ####
Vertices:
Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
Map 2
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out Thu Nov 27 06:05:19 2014 differ
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out?rev=1642026&r1=1642025&r2=1642026&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin9.q.out Thu Nov 27 06:05:19 2014
@@ -106,18 +106,26 @@ TOK_QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Map 2 <- Map 1 (NONE, 0)
#### A masked pattern was here ####
Vertices:
Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
Map 2
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -231,18 +239,26 @@ TOK_QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Map 2 <- Map 1 (NONE, 0)
#### A masked pattern was here ####
Vertices:
Map 1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
Map 2
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator