You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/01/28 19:25:14 UTC

git commit: TEZ-749. Maintain order of vertices as specified by the user (Jonathan Eagles via bikas)

Updated Branches:
  refs/heads/master f8faf4c45 -> a5d481178


TEZ-749. Maintain order of vertices as specified by the user (Jonathan Eagles via bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/a5d48117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/a5d48117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/a5d48117

Branch: refs/heads/master
Commit: a5d4811784661df4829891abb88e1653af75c893
Parents: f8faf4c
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Jan 28 10:25:02 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Jan 28 10:25:02 2014 -0800

----------------------------------------------------------------------
 pom.xml                                         |  5 ++
 tez-api/pom.xml                                 |  4 ++
 .../main/java/org/apache/tez/dag/api/DAG.java   |  9 ++-
 .../org/apache/tez/dag/api/TestDAGPlan.java     | 75 +++++++++++++++++++-
 4 files changed, 86 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 41a5bc0..458d721 100644
--- a/pom.xml
+++ b/pom.xml
@@ -217,6 +217,11 @@
         <version>1.9.5</version>
       </dependency>
       <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-collections4</artifactId>
+        <version>4.0</version>
+      </dependency>
+      <dependency>
         <groupId>commons-cli</groupId>
         <artifactId>commons-cli</artifactId>
         <version>1.2</version>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/tez-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index 075b034..5e10ba4 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -42,6 +42,10 @@
       <artifactId>hadoop-yarn-common</artifactId>
     </dependency>
     <dependency>
+     <groupId>org.apache.commons</groupId>
+     <artifactId>commons-collections4</artifactId>
+    </dependency>
+    <dependency>
      <groupId>com.google.protobuf</groupId>
      <artifactId>protobuf-java</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 84da52c..3840110 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -48,18 +48,17 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
 import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-
+import org.apache.commons.collections4.bidimap.DualLinkedHashBidiMap;
+import org.apache.commons.collections4.BidiMap;
 
 public class DAG { // FIXME rename to Topology
-  final BiMap<String, Vertex> vertices;
+  final BidiMap<String, Vertex> vertices;
   final List<Edge> edges;
   final String name;
   Credentials credentials;
 
   public DAG(String name) {
-    this.vertices = HashBiMap.<String, Vertex>create();
+    this.vertices = new DualLinkedHashBidiMap<String, Vertex>();
     this.edges = new ArrayList<Edge>();
     this.name = name;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/a5d48117/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index 2744c18..1294d9b 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -176,7 +176,7 @@ public class TestDAGPlan {
     assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
         .getUserPayload().toByteArray()));
     assertEquals("input", edgeProto.getEdgeDestination().getClassName());
-             
+
     assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
         .getUserPayload().toByteArray()));
     assertEquals("output", edgeProto.getEdgeSource().getClassName());
@@ -192,7 +192,78 @@ public class TestDAGPlan {
     assertEquals("outputBytes", new String(ob));
     assertEquals("output", edgeProperty.getEdgeSource().getClassName());
   }
-  
+
+  @Test(timeout = 5000)
+  public void userVertexOrderingIsMaintained() {
+    DAG dag = new DAG("testDag");
+    ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
+        setUserPayload("processor1Bytes".getBytes());
+    ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
+        setUserPayload("processor2Bytes".getBytes());
+    ProcessorDescriptor pd3 = new ProcessorDescriptor("processor3").
+        setUserPayload("processor3Bytes".getBytes());
+    Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
+    Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
+    Vertex v3 = new Vertex("v3", pd3, 1, Resource.newInstance(1024, 1));
+    v1.setJavaOpts("").setTaskEnvironment(new HashMap<String, String>())
+        .setTaskLocalResources(new HashMap<String, LocalResource>());
+    v2.setJavaOpts("").setTaskEnvironment(new HashMap<String, String>())
+        .setTaskLocalResources(new HashMap<String, LocalResource>());
+    v3.setJavaOpts("").setTaskEnvironment(new HashMap<String, String>())
+        .setTaskLocalResources(new HashMap<String, LocalResource>());
+
+    InputDescriptor inputDescriptor = new InputDescriptor("input").
+        setUserPayload("inputBytes".getBytes());
+    OutputDescriptor outputDescriptor = new OutputDescriptor("output").
+        setUserPayload("outputBytes".getBytes());
+    Edge edge = new Edge(v1, v2, new EdgeProperty(
+        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+        SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
+
+    dag.addVertex(v1).addVertex(v2).addEdge(edge).addVertex(v3);
+
+    DAGPlan dagProto = dag.createDag(new TezConfiguration());
+
+    assertEquals(3, dagProto.getVertexCount());
+    assertEquals(1, dagProto.getEdgeCount());
+
+    VertexPlan v1Proto = dagProto.getVertex(0);
+    VertexPlan v2Proto = dagProto.getVertex(1);
+    VertexPlan v3Proto = dagProto.getVertex(2);
+    EdgePlan edgeProto = dagProto.getEdge(0);
+
+    assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
+        .getUserPayload().toByteArray()));
+    assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());
+
+    assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
+        .getUserPayload().toByteArray()));
+    assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());
+
+    assertEquals("processor3Bytes", new String(v3Proto.getProcessorDescriptor()
+        .getUserPayload().toByteArray()));
+    assertEquals("processor3", v3Proto.getProcessorDescriptor().getClassName());
+
+    assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
+        .getUserPayload().toByteArray()));
+    assertEquals("input", edgeProto.getEdgeDestination().getClassName());
+
+    assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
+        .getUserPayload().toByteArray()));
+    assertEquals("output", edgeProto.getEdgeSource().getClassName());
+
+    EdgeProperty edgeProperty = DagTypeConverters
+        .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
+
+    byte[] ib = edgeProperty.getEdgeDestination().getUserPayload();
+    assertEquals("inputBytes", new String(ib));
+    assertEquals("input", edgeProperty.getEdgeDestination().getClassName());
+
+    byte[] ob = edgeProperty.getEdgeSource().getUserPayload();
+    assertEquals("outputBytes", new String(ob));
+    assertEquals("output", edgeProperty.getEdgeSource().getClassName());
+  }
+
   @Test (timeout=5000)
   public void testCredentialsSerde() {
     DAG dag = new DAG("testDag");