You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/09/02 01:58:13 UTC
svn commit: r1164301 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server...
Author: acmurthy
Date: Thu Sep 1 23:58:12 2011
New Revision: 1164301
URL: http://svn.apache.org/viewvc?rev=1164301&view=rev
Log:
MAPREDUCE-2735. Add an applications summary log to ResourceManager. Contributed by Thomas Graves.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1164301&r1=1164300&r2=1164301&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Sep 1 23:58:12 2011
@@ -230,6 +230,9 @@ Release 0.23.0 - Unreleased
making shuffle service port to be truely configurable. (Robert Evans via
vinodkv)
+ MAPREDUCE-2735. Add an applications summary log to ResourceManager.
+ (Thomas Graves via acmurthy)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1164301&r1=1164300&r2=1164301&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Thu Sep 1 23:58:12 2011
@@ -34,11 +34,14 @@ import org.apache.hadoop.yarn.security.A
import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.util.StringUtils;
/**
* This class manages the list of applications for the resource manager.
@@ -69,6 +72,86 @@ public class RMAppManager implements Eve
RMConfig.DEFAULT_EXPIRE_APPLICATIONS_COMPLETED_MAX));
}
+ /**
+ * This class is for logging the application summary.
+ */
+ static class ApplicationSummary {
+ static final Log LOG = LogFactory.getLog(ApplicationSummary.class);
+
+ // Escape sequences
+ static final char EQUALS = '=';
+ static final char[] charsToEscape =
+ {StringUtils.COMMA, EQUALS, StringUtils.ESCAPE_CHAR};
+
+ static class SummaryBuilder {
+ final StringBuilder buffer = new StringBuilder();
+
+ // A little optimization for a very common case
+ SummaryBuilder add(String key, long value) {
+ return _add(key, Long.toString(value));
+ }
+
+ <T> SummaryBuilder add(String key, T value) {
+ return _add(key, StringUtils.escapeString(String.valueOf(value),
+ StringUtils.ESCAPE_CHAR, charsToEscape));
+ }
+
+ SummaryBuilder add(SummaryBuilder summary) {
+ if (buffer.length() > 0) buffer.append(StringUtils.COMMA);
+ buffer.append(summary.buffer);
+ return this;
+ }
+
+ SummaryBuilder _add(String key, String value) {
+ if (buffer.length() > 0) buffer.append(StringUtils.COMMA);
+ buffer.append(key).append(EQUALS).append(value);
+ return this;
+ }
+
+ @Override public String toString() {
+ return buffer.toString();
+ }
+ }
+
+ /**
+ * create a summary of the application's runtime.
+ *
+ * @param app {@link RMApp} whose summary is to be created, cannot
+ * be <code>null</code>.
+ */
+ public static SummaryBuilder createAppSummary(RMApp app) {
+ String trackingUrl = "N/A";
+ String host = "N/A";
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ if (attempt != null) {
+ trackingUrl = attempt.getTrackingUrl();
+ host = attempt.getHost();
+ }
+ SummaryBuilder summary = new SummaryBuilder()
+ .add("appId", app.getApplicationId())
+ .add("name", app.getName())
+ .add("user", app.getUser())
+ .add("queue", app.getQueue())
+ .add("state", app.getState())
+ .add("trackingUrl", trackingUrl)
+ .add("appMasterHost", host)
+ .add("startTime", app.getStartTime())
+ .add("finishTime", app.getFinishTime());
+ return summary;
+ }
+
+ /**
+ * Log a summary of the application's runtime.
+ *
+ * @param app {@link RMApp} whose summary is to be logged
+ */
+ public static void logAppSummary(RMApp app) {
+ if (app != null) {
+ LOG.info(createAppSummary(app));
+ }
+ }
+ }
+
protected void setCompletedAppsMax(int max) {
this.completedAppsMax = max;
}
@@ -154,6 +237,7 @@ public class RMAppManager implements Eve
case APP_COMPLETED:
{
addCompletedApp(appID);
+ ApplicationSummary.logAppSummary(rmContext.getRMApps().get(appID));
checkAppNumCompletedLimit();
}
break;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1164301&r1=1164300&r2=1164301&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Thu Sep 1 23:58:12 2011
@@ -36,6 +36,9 @@ public class MockRMApp implements RMApp
RMAppState state = RMAppState.NEW;
int failCount = 0;
ApplicationId id;
+ String url = null;
+ StringBuilder diagnostics = new StringBuilder();
+ RMAppAttempt attempt;
public MockRMApp(int newid, long time, RMAppState newState) {
finish = time;
@@ -58,11 +61,19 @@ public class MockRMApp implements RMApp
return state;
}
+ public void setState(RMAppState state) {
+ this.state = state;
+ }
+
@Override
public String getUser() {
return user;
}
+ public void setUser(String user) {
+ this.user = user;
+ }
+
@Override
public float getProgress() {
return (float) 0.0;
@@ -78,14 +89,26 @@ public class MockRMApp implements RMApp
return queue;
}
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
@Override
public String getName() {
return name;
}
+ public void setName(String name) {
+ this.name = name;
+ }
+
@Override
public RMAppAttempt getCurrentAppAttempt() {
- throw new UnsupportedOperationException("Not supported yet.");
+ return attempt;
+ }
+
+ public void setCurrentAppAttempt(RMAppAttempt attempt) {
+ this.attempt = attempt;
}
@Override
@@ -103,19 +126,35 @@ public class MockRMApp implements RMApp
return finish;
}
+ public void setFinishTime(long time) {
+ this.finish = time;
+ }
+
@Override
public long getStartTime() {
return start;
}
+ public void setStartTime(long time) {
+ this.start = time;
+ }
+
@Override
public String getTrackingUrl() {
- throw new UnsupportedOperationException("Not supported yet.");
+ return url;
+ }
+
+ public void setTrackingUrl(String url) {
+ this.url = url;
}
@Override
public StringBuilder getDiagnostics() {
- throw new UnsupportedOperationException("Not supported yet.");
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diag) {
+ this.diagnostics = new StringBuilder(diag);
}
public void handle(RMAppEvent event) {