You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/22 01:21:46 UTC
[25/29] hadoop git commit: MAPREDUCE-6293. Set job classloader on
uber-job's LocalContainerLauncher event thread. (Sangjin Lee via gera)
MAPREDUCE-6293. Set job classloader on uber-job's LocalContainerLauncher event thread. (Sangjin Lee via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b5d660c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b5d660c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b5d660c
Branch: refs/heads/YARN-2928
Commit: 7b5d660ca7135777735620047ea5d2629319c79f
Parents: 632c409
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Apr 21 11:46:35 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Apr 21 16:16:56 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapred/LocalContainerLauncher.java | 20 ++++++++++++++++++++
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 2 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 +++++++++
5 files changed, 34 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5d660c/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index ffa01fa..0cf5c4b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -337,6 +337,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6238. MR2 can't run local jobs with -libjars command options
which is a regression from MR1 (zxu via rkanter)
+ MAPREDUCE-6293. Set job classloader on uber-job's LocalContainerLauncher
+ event thread. (Sangjin Lee via gera)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5d660c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 218ac83..ffc5326 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -80,6 +80,7 @@ public class LocalContainerLauncher extends AbstractService implements
private final HashSet<File> localizedFiles;
private final AppContext context;
private final TaskUmbilicalProtocol umbilical;
+ private final ClassLoader jobClassLoader;
private ExecutorService taskRunner;
private Thread eventHandler;
private BlockingQueue<ContainerLauncherEvent> eventQueue =
@@ -87,6 +88,12 @@ public class LocalContainerLauncher extends AbstractService implements
public LocalContainerLauncher(AppContext context,
TaskUmbilicalProtocol umbilical) {
+ this(context, umbilical, null);
+ }
+
+ public LocalContainerLauncher(AppContext context,
+ TaskUmbilicalProtocol umbilical,
+ ClassLoader jobClassLoader) {
super(LocalContainerLauncher.class.getName());
this.context = context;
this.umbilical = umbilical;
@@ -94,6 +101,7 @@ public class LocalContainerLauncher extends AbstractService implements
// (TODO/FIXME: pointless to use RPC to talk to self; should create
// LocalTaskAttemptListener or similar: implement umbilical protocol
// but skip RPC stuff)
+ this.jobClassLoader = jobClassLoader;
try {
curFC = FileContext.getFileContext(curDir.toURI());
@@ -133,6 +141,18 @@ public class LocalContainerLauncher extends AbstractService implements
setDaemon(true).setNameFormat("uber-SubtaskRunner").build());
// create and start an event handling thread
eventHandler = new Thread(new EventHandler(), "uber-EventHandler");
+ // if the job classloader is specified, set it onto the event handler as the
+ // thread context classloader so that it can be used by the event handler
+ // as well as the subtask runner threads
+ if (jobClassLoader != null) {
+ LOG.info("Setting " + jobClassLoader +
+ " as the context classloader of thread " + eventHandler.getName());
+ eventHandler.setContextClassLoader(jobClassLoader);
+ } else {
+ // note the current TCCL
+ LOG.info("Context classloader of thread " + eventHandler.getName() +
+ ": " + eventHandler.getContextClassLoader());
+ }
eventHandler.start();
super.serviceStart();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5d660c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 5d3ad5b..9908ea5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -889,7 +889,7 @@ public class MRAppMaster extends CompositeService {
protected void serviceStart() throws Exception {
if (job.isUber()) {
this.containerLauncher = new LocalContainerLauncher(context,
- (TaskUmbilicalProtocol) taskAttemptListener);
+ (TaskUmbilicalProtocol) taskAttemptListener, jobClassLoader);
} else {
this.containerLauncher = new ContainerLauncherImpl(context);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5d660c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index e4b43b5..6b115b3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -378,7 +378,7 @@ public class MRApps extends Apps {
public static void setClassLoader(ClassLoader classLoader,
Configuration conf) {
if (classLoader != null) {
- LOG.info("Setting classloader " + classLoader.getClass().getName() +
+ LOG.info("Setting classloader " + classLoader +
" on the configuration and as the thread context classloader");
conf.setClassLoader(classLoader);
Thread.currentThread().setContextClassLoader(classLoader);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b5d660c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 60e5638..2973c39 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -997,6 +997,15 @@ public class TestMRJobs {
throws IOException, InterruptedException {
super.setup(context);
final Configuration conf = context.getConfiguration();
+ // check if the job classloader is enabled and verify the TCCL
+ if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)) {
+ ClassLoader tccl = Thread.currentThread().getContextClassLoader();
+ if (!(tccl instanceof ApplicationClassLoader)) {
+ throw new IOException("TCCL expected: " +
+ ApplicationClassLoader.class.getName() + ", actual: " +
+ tccl.getClass().getName());
+ }
+ }
final String ioSortMb = conf.get(MRJobConfig.IO_SORT_MB);
if (!TEST_IO_SORT_MB.equals(ioSortMb)) {
throw new IOException("io.sort.mb expected: " + TEST_IO_SORT_MB