You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/01/28 19:25:14 UTC
git commit: TEZ-749. Maintain order of vertices as specified by the
user (Jonathan Eagles via bikas)
Updated Branches:
refs/heads/master f8faf4c45 -> a5d481178
TEZ-749. Maintain order of vertices as specified by the user (Jonathan Eagles via bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a5d48117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a5d48117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a5d48117
Branch: refs/heads/master
Commit: a5d4811784661df4829891abb88e1653af75c893
Parents: f8faf4c
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Jan 28 10:25:02 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Jan 28 10:25:02 2014 -0800
----------------------------------------------------------------------
pom.xml | 5 ++
tez-api/pom.xml | 4 ++
.../main/java/org/apache/tez/dag/api/DAG.java | 9 ++-
.../org/apache/tez/dag/api/TestDAGPlan.java | 75 +++++++++++++++++++-
4 files changed, 86 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 41a5bc0..458d721 100644
--- a/pom.xml
+++ b/pom.xml
@@ -217,6 +217,11 @@
<version>1.9.5</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ <version>4.0</version>
+ </dependency>
+ <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/tez-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index 075b034..5e10ba4 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -42,6 +42,10 @@
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 84da52c..3840110 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -48,18 +48,17 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-
+import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
+import org.apache.commons.collections4.BidiMap;
public class DAG { // FIXME rename to Topology
- final BiMap<String, Vertex> vertices;
+ final BidiMap<String, Vertex> vertices;
final List<Edge> edges;
final String name;
Credentials credentials;
public DAG(String name) {
- this.vertices = HashBiMap.<String, Vertex>create();
+ this.vertices = new DualLinkedHashBidiMap<String, Vertex>();
this.edges = new ArrayList<Edge>();
this.name = name;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index 2744c18..1294d9b 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -176,7 +176,7 @@ public class TestDAGPlan {
assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
.getUserPayload().toByteArray()));
assertEquals("input", edgeProto.getEdgeDestination().getClassName());
-
+
assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
.getUserPayload().toByteArray()));
assertEquals("output", edgeProto.getEdgeSource().getClassName());
@@ -192,7 +192,78 @@ public class TestDAGPlan {
assertEquals("outputBytes", new String(ob));
assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}
-
+
+ @Test(timeout = 5000)
+ public void userVertexOrderingIsMaintained() {
+ DAG dag = new DAG("testDag");
+ ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
+ setUserPayload("processor1Bytes".getBytes());
+ ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
+ setUserPayload("processor2Bytes".getBytes());
+ ProcessorDescriptor pd3 = new ProcessorDescriptor("processor3").
+ setUserPayload("processor3Bytes".getBytes());
+ Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
+ Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
+ Vertex v3 = new Vertex("v3", pd3, 1, Resource.newInstance(1024, 1));
+ v1.setJavaOpts("").setTaskEnvironment(new HashMap<String, String>())
+ .setTaskLocalResources(new HashMap<String, LocalResource>());
+ v2.setJavaOpts("").setTaskEnvironment(new HashMap<String, String>())
+ .setTaskLocalResources(new HashMap<String, LocalResource>());
+ v3.setJavaOpts("").setTaskEnvironment(new HashMap<String, String>())
+ .setTaskLocalResources(new HashMap<String, LocalResource>());
+
+ InputDescriptor inputDescriptor = new InputDescriptor("input").
+ setUserPayload("inputBytes".getBytes());
+ OutputDescriptor outputDescriptor = new OutputDescriptor("output").
+ setUserPayload("outputBytes".getBytes());
+ Edge edge = new Edge(v1, v2, new EdgeProperty(
+ DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
+
+ dag.addVertex(v1).addVertex(v2).addEdge(edge).addVertex(v3);
+
+ DAGPlan dagProto = dag.createDag(new TezConfiguration());
+
+ assertEquals(3, dagProto.getVertexCount());
+ assertEquals(1, dagProto.getEdgeCount());
+
+ VertexPlan v1Proto = dagProto.getVertex(0);
+ VertexPlan v2Proto = dagProto.getVertex(1);
+ VertexPlan v3Proto = dagProto.getVertex(2);
+ EdgePlan edgeProto = dagProto.getEdge(0);
+
+ assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
+ .getUserPayload().toByteArray()));
+ assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());
+
+ assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
+ .getUserPayload().toByteArray()));
+ assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());
+
+ assertEquals("processor3Bytes", new String(v3Proto.getProcessorDescriptor()
+ .getUserPayload().toByteArray()));
+ assertEquals("processor3", v3Proto.getProcessorDescriptor().getClassName());
+
+ assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
+ .getUserPayload().toByteArray()));
+ assertEquals("input", edgeProto.getEdgeDestination().getClassName());
+
+ assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
+ .getUserPayload().toByteArray()));
+ assertEquals("output", edgeProto.getEdgeSource().getClassName());
+
+ EdgeProperty edgeProperty = DagTypeConverters
+ .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
+
+ byte[] ib = edgeProperty.getEdgeDestination().getUserPayload();
+ assertEquals("inputBytes", new String(ib));
+ assertEquals("input", edgeProperty.getEdgeDestination().getClassName());
+
+ byte[] ob = edgeProperty.getEdgeSource().getUserPayload();
+ assertEquals("outputBytes", new String(ob));
+ assertEquals("output", edgeProperty.getEdgeSource().getClassName());
+ }
+
@Test (timeout=5000)
public void testCredentialsSerde() {
DAG dag = new DAG("testDag");