You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by zj...@apache.org on 2014/07/27 03:41:44 UTC
svn commit: r1613744 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/a...
Author: zjshen
Date: Sun Jul 27 01:41:44 2014
New Revision: 1613744
URL: http://svn.apache.org/r1613744
Log:
MAPREDUCE-6002. Made MR task avoid reporting error to AM when the task process is shutting down. Contributed by Wangda Tan.
svn merge --ignore-ancestry -c 1613743 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1613744&r1=1613743&r2=1613744&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Sun Jul 27 01:41:44 2014
@@ -179,6 +179,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5952. LocalContainerLauncher#renameMapOutputForReduce incorrectly
assumes a single dir for mapOutIndex. (Gera Shegalov via kasha)
+ MAPREDUCE-6002. Made MR task avoid reporting error to AM when the task process
+ is shutting down. (Wangda Tan via zjshen)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java?rev=1613744&r1=1613743&r2=1613744&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java Sun Jul 27 01:41:44 2014
@@ -31,6 +31,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import com.google.common.annotations.VisibleForTesting;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSError;
@@ -57,6 +58,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -406,7 +408,9 @@ public class LocalContainerLauncher exte
} catch (FSError e) {
LOG.fatal("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
- umbilical.fsError(classicAttemptID, e.getMessage());
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ umbilical.fsError(classicAttemptID, e.getMessage());
+ }
throw new RuntimeException();
} catch (Exception exception) {
@@ -429,11 +433,13 @@ public class LocalContainerLauncher exte
} catch (Throwable throwable) {
LOG.fatal("Error running local (uberized) 'child' : "
+ StringUtils.stringifyException(throwable));
- Throwable tCause = throwable.getCause();
- String cause = (tCause == null)
- ? throwable.getMessage()
- : StringUtils.stringifyException(tCause);
- umbilical.fatalError(classicAttemptID, cause);
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ Throwable tCause = throwable.getCause();
+ String cause =
+ (tCause == null) ? throwable.getMessage() : StringUtils
+ .stringifyException(tCause);
+ umbilical.fatalError(classicAttemptID, cause);
+ }
throw new RuntimeException();
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1613744&r1=1613743&r2=1613744&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Sun Jul 27 01:41:44 2014
@@ -56,6 +56,7 @@ import org.apache.hadoop.security.Securi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -170,7 +171,9 @@ class YarnChild {
});
} catch (FSError e) {
LOG.fatal("FSError from child", e);
- umbilical.fsError(taskid, e.getMessage());
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ umbilical.fsError(taskid, e.getMessage());
+ }
} catch (Exception exception) {
LOG.warn("Exception running child : "
+ StringUtils.stringifyException(exception));
@@ -195,17 +198,22 @@ class YarnChild {
}
// Report back any failures, for diagnostic purposes
if (taskid != null) {
- umbilical.fatalError(taskid, StringUtils.stringifyException(exception));
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ umbilical.fatalError(taskid,
+ StringUtils.stringifyException(exception));
+ }
}
} catch (Throwable throwable) {
LOG.fatal("Error running child : "
+ StringUtils.stringifyException(throwable));
if (taskid != null) {
- Throwable tCause = throwable.getCause();
- String cause = tCause == null
- ? throwable.getMessage()
- : StringUtils.stringifyException(tCause);
- umbilical.fatalError(taskid, cause);
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ Throwable tCause = throwable.getCause();
+ String cause =
+ tCause == null ? throwable.getMessage() : StringUtils
+ .stringifyException(tCause);
+ umbilical.fatalError(taskid, cause);
+ }
}
} finally {
RPC.stopProxy(umbilical);
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?rev=1613744&r1=1613743&r2=1613744&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Sun Jul 27 01:41:44 2014
@@ -66,6 +66,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
@@ -321,6 +322,11 @@ abstract public class Task implements Wr
protected void reportFatalError(TaskAttemptID id, Throwable throwable,
String logMsg) {
LOG.fatal(logMsg);
+
+ if (ShutdownHookManager.get().isShutdownInProgress()) {
+ return;
+ }
+
Throwable tCause = throwable.getCause();
String cause = tCause == null
? StringUtils.stringifyException(throwable)