You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2013/06/22 01:35:27 UTC
git commit: TEZ-183. Fix Vertex/Task/TaskAttempt toString to use
better formatting. (hitesh)
Updated Branches:
refs/heads/master 6ff6d7e31 -> d7fa15421
TEZ-183. Fix Vertex/Task/TaskAttempt toString to use better formatting. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/d7fa1542
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/d7fa1542
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/d7fa1542
Branch: refs/heads/master
Commit: d7fa154210f9c8540cb2ccec00b65609ab0ed86b
Parents: 6ff6d7e
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Jun 21 16:34:38 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Jun 21 16:34:38 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/dag/records/TezDAGID.java | 25 +++--
.../org/apache/tez/dag/records/TezTaskID.java | 29 ++---
.../org/apache/tez/dag/records/TezVertexID.java | 30 +++--
.../org/apache/tez/dag/records/TestTezIds.java | 111 +++++++++++++++++++
tez-common/src/test/resources/log4j.properties | 19 ++++
5 files changed, 173 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d7fa1542/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
index eff9764..37b6a6e 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java
@@ -25,7 +25,6 @@ import java.text.NumberFormat;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-
/**
* TezDAGID represents the immutable and unique identifier for
* a Tez DAG.
@@ -37,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
* @see ApplicationId
*/
public class TezDAGID extends TezID {
-
+
private ApplicationId applicationId;
public TezDAGID() {
@@ -104,24 +103,30 @@ public class TezDAGID extends TezID {
// DO NOT CHANGE THIS. DAGClient replicates this code to create DAG id string
public static final String DAG = "dag";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ protected static final NumberFormat tezAppIdFormat =
+ NumberFormat.getInstance();
+ protected static final NumberFormat tezDagIdFormat =
+ NumberFormat.getInstance();
+
static {
- idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
+ tezAppIdFormat.setGroupingUsed(false);
+ tezAppIdFormat.setMinimumIntegerDigits(4);
+ tezDagIdFormat.setGroupingUsed(false);
+ tezDagIdFormat.setMinimumIntegerDigits(1);
}
@Override
public String toString() {
return appendTo(new StringBuilder(DAG)).toString();
}
-
+
public static TezDAGID fromString(String dagId) {
try {
String[] split = dagId.split("_");
String rmId = split[1];
- int appId = Integer.parseInt(split[2]);
+ int appId = tezAppIdFormat.parse(split[2]).intValue();
int id;
- id = idFormat.parse(split[3]).intValue();
+ id = tezDagIdFormat.parse(split[3]).intValue();
return new TezDAGID(rmId, appId, id);
} catch (Exception e) {
e.printStackTrace();
@@ -138,9 +143,9 @@ public class TezDAGID extends TezID {
return builder.append(SEPARATOR).
append(applicationId.getClusterTimestamp()).
append(SEPARATOR).
- append(applicationId.getId()).
+ append(tezAppIdFormat.format(applicationId.getId())).
append(SEPARATOR).
- append(idFormat.format(id));
+ append(tezDagIdFormat.format(id));
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d7fa1542/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
index d52e83b..7489ea4 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java
@@ -29,33 +29,34 @@ import org.apache.tez.dag.records.TezVertexID;
/**
- * TaskID represents the immutable and unique identifier for
+ * TaskID represents the immutable and unique identifier for
* a Tez Task. Each TaskID encompasses multiple attempts made to
* execute the Tez Task, each of which are uniquely identified by
* their TezTaskAttemptID.
- *
+ *
* @see TezTaskAttemptID
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class TezTaskID extends TezID {
public static final String TASK = "task";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ protected static final NumberFormat idFormat =
+ NumberFormat.getInstance();
static {
idFormat.setGroupingUsed(false);
idFormat.setMinimumIntegerDigits(6);
}
-
+
private TezVertexID vertexId;
-
+
public TezTaskID() {
vertexId = new TezVertexID();
}
-
+
/**
- * Constructs a TaskID object from given {@link MRxApplicationID}.
- * @param jobId JobID that this tip belongs to
- * @param type the {@link TezTaskType} of the task
+ * Constructs a TaskID object from given {@link MRxApplicationID}.
+ * @param jobId JobID that this tip belongs to
+ * @param type the {@link TezTaskType} of the task
* @param id the tip number
*/
public TezTaskID(TezVertexID vertexId, int id) {
@@ -65,7 +66,7 @@ public class TezTaskID extends TezID {
}
this.vertexId = vertexId;
}
-
+
/** Returns the {@link TezVertexID} object that this task belongs to */
public TezVertexID getVertexID() {
return vertexId;
@@ -91,7 +92,7 @@ public class TezTaskID extends TezID {
else return vertexComp;
}
@Override
- public String toString() {
+ public String toString() {
return appendTo(new StringBuilder(TASK)).toString();
}
@@ -105,12 +106,12 @@ public class TezTaskID extends TezID {
append(SEPARATOR).
append(idFormat.format(id));
}
-
+
@Override
public int hashCode() {
return vertexId.hashCode() * 535013 + id;
}
-
+
@Override
public void readFields(DataInput in) throws IOException {
vertexId.readFields(in);
@@ -122,5 +123,5 @@ public class TezTaskID extends TezID {
vertexId.write(out);
super.write(out);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d7fa1542/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
index 8b2a272..55e93b5 100644
--- a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
+++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java
@@ -41,17 +41,18 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Stable
public class TezVertexID extends TezID {
public static final String VERTEX = "vertex";
- protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ protected static final NumberFormat idFormat =
+ NumberFormat.getInstance();
static {
idFormat.setGroupingUsed(false);
- idFormat.setMinimumIntegerDigits(6);
+ idFormat.setMinimumIntegerDigits(2);
}
-
+
private TezDAGID dagId;
-
+
public TezVertexID() {
}
-
+
/**
* Constructs a TaskID object from given {@link TezDAGID}.
* @param applicationId JobID that this tip belongs to
@@ -65,7 +66,7 @@ public class TezVertexID extends TezID {
}
this.dagId = dagId;
}
-
+
/** Returns the {@link TezDAGID} object that this tip belongs to */
public TezDAGID getDAGId() {
return dagId;
@@ -86,12 +87,12 @@ public class TezVertexID extends TezID {
TezVertexID that = (TezVertexID)o;
return this.dagId.compareTo(that.dagId);
}
-
+
@Override
- public String toString() {
+ public String toString() {
return appendTo(new StringBuilder(VERTEX)).toString();
}
-
+
@Override
public void readFields(DataInput in) throws IOException {
dagId = new TezDAGID();
@@ -111,19 +112,14 @@ public class TezVertexID extends TezID {
* @return the builder that was passed in
*/
protected StringBuilder appendTo(StringBuilder builder) {
- return builder.append(SEPARATOR).
- append(dagId.getApplicationId().getClusterTimestamp()).
- append(SEPARATOR).
- append(dagId.getApplicationId().getId()).
- append(SEPARATOR).
- append(dagId.getId()).
+ return dagId.appendTo(builder).
append(SEPARATOR).
append(idFormat.format(id));
}
-
+
@Override
public int hashCode() {
return dagId.hashCode() * 530017 + id;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d7fa1542/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java b/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java
new file mode 100644
index 0000000..a3d838f
--- /dev/null
+++ b/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.dag.records;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestTezIds {
+
+ private static final Log LOG = LogFactory.getLog(TestTezIds.class);
+
+ private void verifyDagInfo(String[] splits, TezDAGID dagId) {
+ Assert.assertEquals(dagId.getApplicationId().getClusterTimestamp(),
+ Long.valueOf(splits[1]).longValue());
+ Assert.assertEquals(dagId.getApplicationId().getId(),
+ Integer.valueOf(splits[2]).intValue());
+ Assert.assertEquals(dagId.getId(),
+ Integer.valueOf(splits[3]).intValue());
+ }
+
+ private void verifyVertexInfo(String[] splits, TezVertexID vId) {
+ verifyDagInfo(splits, vId.getDAGId());
+ Assert.assertEquals(vId.getId(),
+ Integer.valueOf(splits[4]).intValue());
+ }
+
+ private void verifyTaskInfo(String[] splits, TezTaskID tId) {
+ verifyVertexInfo(splits, tId.getVertexID());
+ Assert.assertEquals(tId.getId(),
+ Integer.valueOf(splits[5]).intValue());
+ }
+
+ private void verifyAttemptInfo(String[] splits, TezTaskAttemptID taId) {
+ verifyTaskInfo(splits, taId.getTaskID());
+ Assert.assertEquals(taId.getId(),
+ Integer.valueOf(splits[6]).intValue());
+ }
+
+ private void verifyDagId(String dagIdStr, TezDAGID dagId) {
+ String[] splits = dagIdStr.split("_");
+ Assert.assertEquals(4, splits.length);
+ verifyDagInfo(splits, dagId);
+ }
+
+ private void verifyVertexId(String vIdStr, TezVertexID vId) {
+ String[] splits = vIdStr.split("_");
+ Assert.assertEquals(5, splits.length);
+ verifyVertexInfo(splits, vId);
+ }
+
+ private void verifyTaskId(String tIdStr, TezTaskID tId) {
+ String[] splits = tIdStr.split("_");
+ Assert.assertEquals(6, splits.length);
+ verifyTaskInfo(splits, tId);
+ }
+
+ private void verifyAttemptId(String taIdStr, TezTaskAttemptID taId) {
+ String[] splits = taIdStr.split("_");
+ Assert.assertEquals(7, splits.length);
+ verifyAttemptInfo(splits, taId);
+ }
+
+ @Test
+ public void testIdStringify() {
+ ApplicationId appId = ApplicationId.newInstance(9999, 72);
+ TezDAGID dagId = new TezDAGID(appId, 1);
+ TezVertexID vId = new TezVertexID(dagId, 35);
+ TezTaskID tId = new TezTaskID(vId, 389);
+ TezTaskAttemptID taId = new TezTaskAttemptID(tId, 2);
+
+ String dagIdStr = dagId.toString();
+ String vIdStr = vId.toString();
+ String tIdStr = tId.toString();
+ String taIdStr = taId.toString();
+
+ LOG.info("DAG ID:" + dagIdStr);
+ LOG.info("Vertex ID:" + vIdStr);
+ LOG.info("Task ID:" + tIdStr);
+ LOG.info("Attempt ID:" + taIdStr);
+
+ Assert.assertTrue(dagIdStr.startsWith("dag"));
+ Assert.assertTrue(vIdStr.startsWith("vertex"));
+ Assert.assertTrue(tIdStr.startsWith("task"));
+ Assert.assertTrue(taIdStr.startsWith("attempt"));
+
+ verifyDagId(dagIdStr, dagId);
+ verifyVertexId(vIdStr, vId);
+ verifyTaskId(tIdStr, tId);
+ verifyAttemptId(taIdStr, taId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/d7fa1542/tez-common/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tez-common/src/test/resources/log4j.properties b/tez-common/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/tez-common/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n