You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by rk...@apache.org on 2016/12/28 23:28:48 UTC
hadoop git commit: YARN-4882. Change the log level to DEBUG for
recovering completed applications (templedf via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/trunk 9ca54f481 -> f216276d2
YARN-4882. Change the log level to DEBUG for recovering completed applications (templedf via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f216276d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f216276d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f216276d
Branch: refs/heads/trunk
Commit: f216276d2164c6564632c571fd3adbb03bc8b3e4
Parents: 9ca54f4
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Dec 28 15:21:52 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Dec 28 15:21:52 2016 -0800
----------------------------------------------------------------------
.../server/resourcemanager/RMAppManager.java | 13 ++++++--
.../server/resourcemanager/rmapp/RMAppImpl.java | 31 +++++++++++++++-----
.../rmapp/attempt/RMAppAttemptImpl.java | 27 +++++++++++++----
3 files changed, 57 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f216276d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 4d628ee..bc21952 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -488,8 +488,17 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
Map<ApplicationId, ApplicationStateData> appStates =
state.getApplicationState();
LOG.info("Recovering " + appStates.size() + " applications");
- for (ApplicationStateData appState : appStates.values()) {
- recoverApplication(appState, state);
+
+ int count = 0;
+
+ try {
+ for (ApplicationStateData appState : appStates.values()) {
+ recoverApplication(appState, state);
+ count += 1;
+ }
+ } finally {
+ LOG.info("Successfully recovered " + count + " out of "
+ + appStates.size() + " applications");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f216276d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index a647969..4db595e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -128,6 +128,10 @@ public class RMAppImpl implements RMApp, Recoverable {
private static final EnumSet<RMAppState> COMPLETED_APP_STATES =
EnumSet.of(RMAppState.FINISHED, RMAppState.FINISHING, RMAppState.FAILED,
RMAppState.KILLED, RMAppState.FINAL_SAVING, RMAppState.KILLING);
+ private static final String STATE_CHANGE_MESSAGE =
+ "%s State change from %s to %s on event = %s";
+ private static final String RECOVERY_MESSAGE =
+ "Recovering app: %s with %d attempts and final state = %s";
// Immutable fields
private final ApplicationId applicationId;
@@ -905,9 +909,16 @@ public class RMAppImpl implements RMApp, Recoverable {
/* TODO fail the application on the failed transition */
}
- if (oldState != getState()) {
- LOG.info(appID + " State change from " + oldState + " to "
- + getState() + " on event=" + event.getType());
+ // Log at INFO if we're not recovering or not in a terminal state.
+ // Log at DEBUG otherwise.
+ if ((oldState != getState()) &&
+ (((recoveredFinalState == null)) ||
+ (event.getType() != RMAppEventType.RECOVER))) {
+ LOG.info(String.format(STATE_CHANGE_MESSAGE, appID, oldState,
+ getState(), event.getType()));
+ } else if ((oldState != getState()) && LOG.isDebugEnabled()) {
+ LOG.debug(String.format(STATE_CHANGE_MESSAGE, appID, oldState,
+ getState(), event.getType()));
}
} finally {
this.writeLock.unlock();
@@ -919,9 +930,15 @@ public class RMAppImpl implements RMApp, Recoverable {
ApplicationStateData appState =
state.getApplicationState().get(getApplicationId());
this.recoveredFinalState = appState.getState();
- LOG.info("Recovering app: " + getApplicationId() + " with " +
- + appState.getAttemptCount() + " attempts and final state = "
- + this.recoveredFinalState );
+
+ if (recoveredFinalState == null) {
+ LOG.info(String.format(RECOVERY_MESSAGE, getApplicationId(),
+ appState.getAttemptCount(), "NONE"));
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format(RECOVERY_MESSAGE, getApplicationId(),
+ appState.getAttemptCount(), recoveredFinalState));
+ }
+
this.diagnostics.append(null == appState.getDiagnostics() ? "" : appState
.getDiagnostics());
this.storedFinishTime = appState.getFinishTime();
@@ -2029,4 +2046,4 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setApplicationPriority(Priority applicationPriority) {
this.applicationPriority = applicationPriority;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f216276d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index e3d1855..f7ae488 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -114,6 +114,10 @@ import com.google.common.annotations.VisibleForTesting;
@SuppressWarnings({"unchecked", "rawtypes"})
public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
+ private static final String STATE_CHANGE_MESSAGE =
+ "%s State change from %s to %s on event = %s";
+ private static final String RECOVERY_MESSAGE =
+ "Recovering attempt: %s with final state = %s";
private static final Log LOG = LogFactory.getLog(RMAppAttemptImpl.class);
@@ -867,9 +871,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
/* TODO fail the application on the failed transition */
}
- if (oldState != getAppAttemptState()) {
- LOG.info(appAttemptID + " State change from " + oldState + " to "
- + getAppAttemptState());
+ // Log at INFO if we're not recovering or not in a terminal state.
+ // Log at DEBUG otherwise.
+ if ((oldState != getAppAttemptState()) &&
+ ((recoveredFinalState == null) ||
+ (event.getType() != RMAppAttemptEventType.RECOVER))) {
+ LOG.info(String.format(STATE_CHANGE_MESSAGE, appAttemptID, oldState,
+ getAppAttemptState(), event.getType()));
+ } else if ((oldState != getAppAttemptState()) && LOG.isDebugEnabled()) {
+ LOG.debug(String.format(STATE_CHANGE_MESSAGE, appAttemptID, oldState,
+ getAppAttemptState(), event.getType()));
}
} finally {
this.writeLock.unlock();
@@ -906,8 +917,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ApplicationAttemptStateData attemptState =
appState.getAttempt(getAppAttemptId());
assert attemptState != null;
- LOG.info("Recovering attempt: " + getAppAttemptId() + " with final state: "
- + attemptState.getState());
+
+ if (attemptState.getState() == null) {
+ LOG.info(String.format(RECOVERY_MESSAGE, getAppAttemptId(), "NONE"));
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format(RECOVERY_MESSAGE, getAppAttemptId(),
+ attemptState.getState()));
+ }
+
diagnostics.append("Attempt recovered after RM restart");
diagnostics.append(attemptState.getDiagnostics());
this.amContainerExitStatus = attemptState.getAMContainerExitStatus();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org