You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by zj...@apache.org on 2015/04/30 06:53:41 UTC
tez git commit: TEZ-2221. VertexGroup name should be unqiue (zjffdu)
Repository: tez
Updated Branches:
refs/heads/master 3894c5ec6 -> 2382f09ff
TEZ-2221. VertexGroup name should be unqiue (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2382f09f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2382f09f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2382f09f
Branch: refs/heads/master
Commit: 2382f09ffaabe1be6ede8203025027198c3e6d2b
Parents: 3894c5e
Author: Jeff Zhang <zj...@apache.org>
Authored: Thu Apr 30 12:53:30 2015 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Apr 30 12:53:30 2015 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../main/java/org/apache/tez/dag/api/DAG.java | 22 +++++++++++++++++++-
.../org/apache/tez/dag/api/VertexGroup.java | 10 ++-------
.../java/org/apache/tez/dag/api/TestDAG.java | 16 ++++++++++++++
4 files changed, 40 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cfdc679..4b031ca 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -325,6 +325,7 @@ TEZ-UI CHANGES (TEZ-8):
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-2221. VertexGroup name should be unqiue
TEZ-1521. VertexDataMovementEventsGeneratedEvent may be logged twice in recovery log
TEZ-2348. EOF exception during UnorderedKVReader.next().
TEZ-1560. Invalid state machine handling for V_SOURCE_VERTEX_RECOVERED in recovery.
http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 79e8759..92f865d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -60,7 +60,9 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -83,6 +85,9 @@ public class DAG {
final Collection<URI> urisForCredentials = new HashSet<URI>();
Credentials credentials = new Credentials();
Set<VertexGroup> vertexGroups = Sets.newHashSet();
+ // to verify the vertex Group memberSet should be unique
+ private Set<Set<String>> vertexGroupMemberSets = Sets.newHashSet();
+
Set<GroupInputEdge> groupInputEdges = Sets.newHashSet();
private DAGAccessControls dagAccessControls;
@@ -176,11 +181,26 @@ public class DAG {
* @return {@link DAG}
*/
public synchronized VertexGroup createVertexGroup(String name, Vertex... members) {
+ // vertex group member set should be unique
+ Collection<String> memberNames =
+ Collections2.transform(Lists.newArrayList(members), new Function<Vertex, String>() {
+ @Override
+ public String apply(Vertex v) {
+ return v.getName();
+ }
+ });
+ if (!vertexGroupMemberSets.add(Sets.newHashSet(memberNames))){
+ throw new IllegalStateException(
+ "VertexGroup " + memberNames + " already defined as another group!");
+ }
+
+ // vertex group name should be unique.
VertexGroup uv = new VertexGroup(name, members);
- if (!vertexGroups.add(uv)){
+ if (!vertexGroups.add(uv)) {
throw new IllegalStateException(
"VertexGroup " + name + " already defined!");
}
+
return uv;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
index 84a822f..8041b34 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java
@@ -108,7 +108,7 @@ public class VertexGroup {
@Override
public String toString() {
- return "[ VertexGroup: " + groupInfo.getGroupName() + "]";
+ return "[ VertexGroup: " + groupInfo.getGroupName() + "]" + ", members=" + groupInfo.members;
}
GroupInfo getGroupInfo() {
@@ -129,8 +129,6 @@ public class VertexGroup {
int result = 1;
result = prime * result
+ ((groupInfo.groupName == null) ? 0 : groupInfo.groupName.hashCode());
- result = prime * result
- + ((groupInfo.members == null) ? 0 : groupInfo.members.hashCode());
return result;
}
@@ -148,11 +146,7 @@ public class VertexGroup {
return false;
} else if (!groupInfo.groupName.equals(other.groupInfo.groupName))
return false;
- if (groupInfo.members == null) {
- if (other.groupInfo.members != null)
- return false;
- } else if (!groupInfo.members.equals(other.groupInfo.members))
- return false;
+
return true;
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2382f09f/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
index 0590907..245d9c0 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java
@@ -82,6 +82,8 @@ public class TestDAG {
dummyTaskCount, dummyTaskResource);
Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
+ Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Processor"),
+ dummyTaskCount, dummyTaskResource);
DAG dag = DAG.create("testDAG");
dag.createVertexGroup("group_1", v1,v2);
@@ -90,8 +92,22 @@ public class TestDAG {
Assert.fail("should fail it due to duplicated VertexGroups");
} catch (Exception e) {
e.printStackTrace();
+ Assert.assertEquals("VertexGroup [v1, v2] already defined as another group!", e.getMessage());
+ }
+ try {
+ dag.createVertexGroup("group_1", v2, v3);
+ Assert.fail("should fail it due to duplicated VertexGroups");
+ } catch (Exception e) {
+ e.printStackTrace();
Assert.assertEquals("VertexGroup group_1 already defined!", e.getMessage());
}
+ try {
+ dag.createVertexGroup("group_2", v1, v2);
+ Assert.fail("should fail it due to duplicated VertexGroups");
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.assertEquals("VertexGroup [v1, v2] already defined as another group!", e.getMessage());
+ }
}
@Test(timeout = 5000)