You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/09/30 23:36:42 UTC
svn commit: r1527791 - in /hive/branches/tez: eclipse-templates/ ivy/ ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/
ql/src/java/org/apache/hadoop/hive/ql/plan/
ql/src/test/org/apache/hadoop/hive/ql/plan/
Author: gunther
Date: Mon Sep 30 21:36:42 2013
New Revision: 1527791
URL: http://svn.apache.org/r1527791
Log:
HIVE-5387: Need to create edge properties for hive on tez (Vikram Dixit K via Gunther Hagleitner)
Modified:
hive/branches/tez/eclipse-templates/.classpath
hive/branches/tez/ivy/libraries.properties
hive/branches/tez/ql/ivy.xml
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
Modified: hive/branches/tez/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/tez/eclipse-templates/.classpath?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/eclipse-templates/.classpath (original)
+++ hive/branches/tez/eclipse-templates/.classpath Mon Sep 30 21:36:42 2013
@@ -75,6 +75,7 @@
<classpathentry kind="lib" path="build/ivy/lib/default/commons-collections-@commons-collections.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/bonecp-@BoneCP.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/commons-pool-@commons-pool.version@.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/default/commons-lang3-@commons-lang3.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/slf4j-api-@slf4j-api.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/slf4j-log4j12-@slf4j-log4j12.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/JavaEWAH-@javaewah.version@.jar"/>
Modified: hive/branches/tez/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/ivy/libraries.properties?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ivy/libraries.properties (original)
+++ hive/branches/tez/ivy/libraries.properties Mon Sep 30 21:36:42 2013
@@ -40,6 +40,7 @@ commons-lang.version=2.4
commons-logging.version=1.0.4
commons-logging-api.version=1.0.4
commons-pool.version=1.5.4
+commons-lang3.version=3.1
derby.version=10.4.2.0
guava.version=11.0.2
hbase.version=0.94.6.1
Modified: hive/branches/tez/ql/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/ivy.xml?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/ivy.xml (original)
+++ hive/branches/tez/ql/ivy.xml Mon Sep 30 21:36:42 2013
@@ -66,7 +66,8 @@
<dependency org="org.json" name="json" rev="${json.version}"/>
<dependency org="commons-collections" name="commons-collections" rev="${commons-collections.version}"/>
<dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
- transitive="false"/>
+ transitive="false"/>
+ <dependency org="org.apache.commons" name="commons-lang3" rev="${commons-lang3.version}"/>
<dependency org="com.googlecode.javaewah" name="JavaEWAH" rev="${javaewah.version}"/>
<dependency org="javolution" name="javolution" rev="${javolution.version}"/>
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Mon Sep 30 21:36:42 2013
@@ -174,10 +174,8 @@ public class TezTask extends Task<TezWor
for (BaseWork v: work.getChildren(w)) {
assert workToVertex.containsKey(v);
Edge e = null;
- EdgeType edgeType = EdgeType.SIMPLE_EDGE;
- if (work.isBroadCastEdge(w, v)) {
- edgeType = EdgeType.BROADCAST_EDGE;
- }
+
+ EdgeType edgeType = work.getEdgeProperty(w, v);
e = DagUtils.createEdge(wxConf, wx, workToConf.get(v), workToVertex.get(v), edgeType);
dag.addEdge(e);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java Mon Sep 30 21:36:42 2013
@@ -25,6 +25,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -49,7 +51,8 @@ public class TezWork extends AbstractOpe
private final Set<BaseWork> leaves = new HashSet<BaseWork>();
private final Map<BaseWork, List<BaseWork>> workGraph = new HashMap<BaseWork, List<BaseWork>>();
private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new HashMap<BaseWork, List<BaseWork>>();
- private final Map<BaseWork, List<BaseWork>> broadcastEdge = new HashMap<BaseWork, List<BaseWork>>();
+ private final Map<Pair<BaseWork, BaseWork>, EdgeType> edgeProperties =
+ new HashMap<Pair<BaseWork, BaseWork>, EdgeType>();
/**
* getAllWork returns a topologically sorted list of BaseWork
@@ -95,7 +98,6 @@ public class TezWork extends AbstractOpe
}
workGraph.put(w, new LinkedList<BaseWork>());
invertedWorkGraph.put(w, new LinkedList<BaseWork>());
- broadcastEdge.put(w, new LinkedList<BaseWork>());
roots.add(w);
leaves.add(w);
}
@@ -109,13 +111,8 @@ public class TezWork extends AbstractOpe
invertedWorkGraph.get(b).add(a);
roots.remove(b);
leaves.remove(a);
- switch (edgeType) {
- case BROADCAST_EDGE:
- broadcastEdge.get(a).add(b);
- break;
- default:
- break;
- }
+ ImmutablePair workPair = new ImmutablePair(a, b);
+ edgeProperties.put(workPair, edgeType);
}
/**
@@ -201,11 +198,7 @@ public class TezWork extends AbstractOpe
invertedWorkGraph.remove(work);
}
- // checks if a and b need a broadcast edge between them
- public boolean isBroadCastEdge(BaseWork a, BaseWork b) {
- if ((broadcastEdge.get(a).contains(b)) || (broadcastEdge.get(b).contains(a))) {
- return true;
- }
- return false;
+ public EdgeType getEdgeProperty(BaseWork a, BaseWork b) {
+ return edgeProperties.get(new ImmutablePair(a,b));
}
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java Mon Sep 30 21:36:42 2013
@@ -77,6 +77,8 @@ public class TestTezWork {
Assert.assertEquals(work.getParents(w).size(), 0);
Assert.assertEquals(work.getChildren(w).size(), 0);
}
+
+ Assert.assertEquals(work.getEdgeProperty(parent, child), EdgeType.SIMPLE_EDGE);
}
@Test
@@ -99,6 +101,8 @@ public class TestTezWork {
Assert.assertEquals(work.getParents(w).size(), 0);
Assert.assertEquals(work.getChildren(w).size(), 0);
}
+
+ Assert.assertEquals(work.getEdgeProperty(parent, child), EdgeType.BROADCAST_EDGE);
}
@Test