You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/10/03 20:04:56 UTC
git commit: YARN-2562. Changed ContainerId#toString() to be more
readable. Contributed by Tsuyoshi OZAWA (cherry picked from commit
eb6ce5e97c6a379caf9de0b34cc7770c4447bb1a)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9ebff016c -> 1c58ab920
YARN-2562. Changed ContainerId#toString() to be more readable. Contributed by Tsuyoshi OZAWA
(cherry picked from commit eb6ce5e97c6a379caf9de0b34cc7770c4447bb1a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c58ab92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c58ab92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c58ab92
Branch: refs/heads/branch-2
Commit: 1c58ab92063d7e44be4fc2dca64d5e1285b323de
Parents: 9ebff01
Author: Jian He <ji...@apache.org>
Authored: Fri Oct 3 11:04:02 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Oct 3 11:04:46 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/api/records/ContainerId.java | 43 +++++++++++++++-----
.../apache/hadoop/yarn/api/TestContainerId.java | 6 ++-
.../hadoop/yarn/util/TestConverterUtils.java | 6 ++-
4 files changed, 44 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c58ab92/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f6e849c..628dfa2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -274,6 +274,9 @@ Release 2.6.0 - UNRELEASED
YARN-2627. Added the info logs of attemptFailuresValidityInterval and number
of previous failed attempts. (Xuan Gong via zjshen)
+ YARN-2562. Changed ContainerId#toString() to be more readable. (Tsuyoshi
+ OZAWA via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c58ab92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
index d03a439..b7c03ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class ContainerId implements Comparable<ContainerId>{
private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
private static final String CONTAINER_PREFIX = "container";
+ private static final String EPOCH_PREFIX = "e";
@Private
@Unstable
@@ -158,10 +159,24 @@ public abstract class ContainerId implements Comparable<ContainerId>{
}
}
+ /**
+ * @return A string representation of containerId. The format is
+ * container_e*epoch*_*clusterTimestamp*_*appId*_*attemptId*_*containerId*
+ * when epoch is larger than 0
+ * (e.g. container_e17_1410901177871_0001_01_000005).
+ * *epoch* is increased when RM restarts or fails over.
+ * When epoch is 0, epoch is omitted
+ * (e.g. container_1410901177871_0001_01_000005).
+ */
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("container_");
+ sb.append(CONTAINER_PREFIX + "_");
+ long epoch = getContainerId() >> 40;
+ if (epoch > 0) {
+ sb.append(EPOCH_PREFIX)
+ .append(appAttemptIdAndEpochFormat.get().format(epoch)).append("_");;
+ }
ApplicationId appId = getApplicationAttemptId().getApplicationId();
sb.append(appId.getClusterTimestamp()).append("_");
sb.append(ApplicationId.appIdFormat.get().format(appId.getId()))
@@ -170,10 +185,6 @@ public abstract class ContainerId implements Comparable<ContainerId>{
appAttemptIdAndEpochFormat.get().format(
getApplicationAttemptId().getAttemptId())).append("_");
sb.append(containerIdFormat.get().format(0xffffffffffL & getContainerId()));
- long epoch = getContainerId() >> 40;
- if (epoch > 0) {
- sb.append("_").append(appAttemptIdAndEpochFormat.get().format(epoch));
- }
return sb.toString();
}
@@ -186,12 +197,19 @@ public abstract class ContainerId implements Comparable<ContainerId>{
+ containerIdStr);
}
try {
- ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
- long id = Long.parseLong(it.next());
+ String epochOrClusterTimestampStr = it.next();
long epoch = 0;
- if (it.hasNext()) {
- epoch = Integer.parseInt(it.next());
+ ApplicationAttemptId appAttemptID = null;
+ if (epochOrClusterTimestampStr.startsWith(EPOCH_PREFIX)) {
+ String epochStr = epochOrClusterTimestampStr;
+ epoch = Integer.parseInt(epochStr.substring(EPOCH_PREFIX.length()));
+ appAttemptID = toApplicationAttemptId(it);
+ } else {
+ String clusterTimestampStr = epochOrClusterTimestampStr;
+ long clusterTimestamp = Long.parseLong(clusterTimestampStr);
+ appAttemptID = toApplicationAttemptId(clusterTimestamp, it);
}
+ long id = Long.parseLong(it.next());
long cid = (epoch << 40) | id;
ContainerId containerId = ContainerId.newInstance(appAttemptID, cid);
return containerId;
@@ -203,7 +221,12 @@ public abstract class ContainerId implements Comparable<ContainerId>{
private static ApplicationAttemptId toApplicationAttemptId(
Iterator<String> it) throws NumberFormatException {
- ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+ return toApplicationAttemptId(Long.parseLong(it.next()), it);
+ }
+
+ private static ApplicationAttemptId toApplicationAttemptId(
+ long clusterTimestamp, Iterator<String> it) throws NumberFormatException {
+ ApplicationId appId = ApplicationId.newInstance(clusterTimestamp,
Integer.parseInt(it.next()));
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c58ab92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
index c11fd96..2259294 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
@@ -63,13 +63,15 @@ public class TestContainerId {
Assert.assertEquals(999799999997L, 0xffffffffffL & c7.getContainerId());
Assert.assertEquals(3, c7.getContainerId() >> 40);
Assert.assertEquals(
- "container_" + ts + "_36473_4365472_999799999997_03", c7.toString());
+ "container_e03_" + ts + "_36473_4365472_999799999997",
+ c7.toString());
ContainerId c8 = newContainerId(36473, 4365472, ts, 844424930131965L);
Assert.assertEquals(1099511627773L, 0xffffffffffL & c8.getContainerId());
Assert.assertEquals(767, c8.getContainerId() >> 40);
Assert.assertEquals(
- "container_" + ts + "_36473_4365472_1099511627773_767", c8.toString());
+ "container_e767_" + ts + "_36473_4365472_1099511627773",
+ c8.toString());
}
public static ContainerId newContainerId(int appId, int appAttemptId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c58ab92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
index f7e7fe4..824e6c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
@@ -67,14 +67,16 @@ public class TestConverterUtils {
ContainerId id2 =
TestContainerId.newContainerId(36473, 4365472, ts, 4298334883325L);
String cid2 = ConverterUtils.toString(id2);
- assertEquals("container_" + ts + "_36473_4365472_999799999997_03", cid2);
+ assertEquals(
+ "container_e03_" + ts + "_36473_4365472_999799999997", cid2);
ContainerId gen2 = ConverterUtils.toContainerId(cid2);
assertEquals(gen2.toString(), id2.toString());
ContainerId id3 =
TestContainerId.newContainerId(36473, 4365472, ts, 844424930131965L);
String cid3 = ConverterUtils.toString(id3);
- assertEquals("container_" + ts + "_36473_4365472_1099511627773_767", cid3);
+ assertEquals(
+ "container_e767_" + ts + "_36473_4365472_1099511627773", cid3);
ContainerId gen3 = ConverterUtils.toContainerId(cid3);
assertEquals(gen3.toString(), id3.toString());
}