You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by do...@apache.org on 2015/10/30 02:44:06 UTC
incubator-reef git commit: [REEF-882] Properly report the message of
an Exception causing a failure on YARN to the RM
Repository: incubator-reef
Updated Branches:
refs/heads/master 49b16128e -> f10e8d35d
[REEF-882] Properly report the message of an Exception causing a failure on YARN to the RM
This addressed the issue by
* Inverting null check in RuntimeClock to assign when uninitialized.
* Changing assumptions of unregistering YARN application as REEF failure
when there are Driver Exceptions.
JIRA:
[REEF-882](https://issues.apache.org/jira/browse/REEF-882)
Pull Request:
Closes #597
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/f10e8d35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/f10e8d35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/f10e8d35
Branch: refs/heads/master
Commit: f10e8d35d72a3fa5c454b4c6db8c5ef400d693a7
Parents: 49b1612
Author: Andrew Chung <af...@gmail.com>
Authored: Wed Oct 28 14:44:42 2015 -0700
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Fri Oct 30 10:42:13 2015 +0900
----------------------------------------------------------------------
.../reef/runtime/yarn/driver/YarnContainerManager.java | 13 ++++++++++---
.../apache/reef/wake/time/runtime/RuntimeClock.java | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f10e8d35/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
index a886eb1..51a8022 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
@@ -20,6 +20,7 @@ package org.apache.reef.runtime.yarn.driver;
import com.google.protobuf.ByteString;
import org.apache.commons.collections.ListUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.*;
@@ -32,7 +33,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.reef.driver.ProgressProvider;
import org.apache.reef.driver.parameters.JobSubmissionDirectory;
-import org.apache.reef.exception.DriverFatalRuntimeException;
import org.apache.reef.proto.ReefServiceProtos;
import org.apache.reef.runtime.common.driver.DriverStatusManager;
import org.apache.reef.runtime.common.driver.resourcemanager.NodeDescriptorEventImpl;
@@ -285,9 +285,16 @@ final class YarnContainerManager
if (exception == null) {
this.resourceManager.unregisterApplicationMaster(
FinalApplicationStatus.SUCCEEDED, null, null);
- } else if (exception instanceof DriverFatalRuntimeException) {
+ } else {
+ // Note: We don't allow RM to restart our applications if it's an application level failure.
+ // If applications are to be long-running, they should catch Exceptions before the REEF level
+ // instead of relying on the RM restart mechanism.
+ // For this case, we make a strong assumption that REEF does not allow its own unhandled Exceptions
+ // to leak to this stage.
+ final String failureMsg = String.format("Application failed due to:%n%s%n" +
+ "With stack trace:%n%s", exception.getMessage(), ExceptionUtils.getStackTrace(exception));
this.resourceManager.unregisterApplicationMaster(
- FinalApplicationStatus.FAILED, null, null);
+ FinalApplicationStatus.FAILED, failureMsg, null);
}
this.resourceManager.close();
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/f10e8d35/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/time/runtime/RuntimeClock.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/time/runtime/RuntimeClock.java b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/time/runtime/RuntimeClock.java
index 01f6744..00ff0a0 100644
--- a/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/time/runtime/RuntimeClock.java
+++ b/lang/java/reef-wake/wake/src/main/java/org/apache/reef/wake/time/runtime/RuntimeClock.java
@@ -114,7 +114,7 @@ public final class RuntimeClock implements Clock {
this.schedule.add(new StopTime(timer.getCurrent()));
this.schedule.notifyAll();
this.closed = true;
- if (this.stoppedOnException != null) {
+ if (this.stoppedOnException == null) {
this.stoppedOnException = stopOnException;
}
}