You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/01/13 00:58:45 UTC
hadoop git commit: MAPREDUCE-6473. Job submission can take a long
time during Cluster initialization. Contributed by Kuhu Shukla (cherry picked
from commit f657b54281d517c69df0a23b97b47befa77d1dc2)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2605e78e4 -> 0af01092b
MAPREDUCE-6473. Job submission can take a long time during Cluster initialization. Contributed by Kuhu Shukla
(cherry picked from commit f657b54281d517c69df0a23b97b47befa77d1dc2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0af01092
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0af01092
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0af01092
Branch: refs/heads/branch-2
Commit: 0af01092b63fc804e56052a8fcaac1045e402960
Parents: 2605e78
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Jan 12 23:56:23 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Jan 12 23:58:01 2016 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../org/apache/hadoop/mapreduce/Cluster.java | 67 ++++++++++++--------
2 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af01092/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index caa0d77..ef67240 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -171,6 +171,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6394. Speed up Task processing loop in HsTasksBlock#render()
(Ray Chiang via jlowe)
+ MAPREDUCE-6473. Job submission can take a long time during Cluster
+ initialization (Kuhu Shukla via jlowe)
+
BUG FIXES
MAPREDUCE-6314. TestPipeApplication fails on trunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af01092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
index fc228e6..9563c0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
@@ -66,7 +66,23 @@ public class Cluster {
private static ServiceLoader<ClientProtocolProvider> frameworkLoader =
ServiceLoader.load(ClientProtocolProvider.class);
-
+ private volatile List<ClientProtocolProvider> providerList = null;
+
+ private void initProviderList() {
+ if (providerList == null) {
+ synchronized (frameworkLoader) {
+ if (providerList == null) {
+ List<ClientProtocolProvider> localProviderList =
+ new ArrayList<ClientProtocolProvider>();
+ for (ClientProtocolProvider provider : frameworkLoader) {
+ localProviderList.add(provider);
+ }
+ providerList = localProviderList;
+ }
+ }
+ }
+ }
+
static {
ConfigUtil.loadResources();
}
@@ -85,34 +101,31 @@ public class Cluster {
private void initialize(InetSocketAddress jobTrackAddr, Configuration conf)
throws IOException {
- synchronized (frameworkLoader) {
- for (ClientProtocolProvider provider : frameworkLoader) {
- LOG.debug("Trying ClientProtocolProvider : "
- + provider.getClass().getName());
- ClientProtocol clientProtocol = null;
- try {
- if (jobTrackAddr == null) {
- clientProtocol = provider.create(conf);
- } else {
- clientProtocol = provider.create(jobTrackAddr, conf);
- }
+ initProviderList();
+ for (ClientProtocolProvider provider : providerList) {
+ LOG.debug("Trying ClientProtocolProvider : "
+ + provider.getClass().getName());
+ ClientProtocol clientProtocol = null;
+ try {
+ if (jobTrackAddr == null) {
+ clientProtocol = provider.create(conf);
+ } else {
+ clientProtocol = provider.create(jobTrackAddr, conf);
+ }
- if (clientProtocol != null) {
- clientProtocolProvider = provider;
- client = clientProtocol;
- LOG.debug("Picked " + provider.getClass().getName()
- + " as the ClientProtocolProvider");
- break;
- }
- else {
- LOG.debug("Cannot pick " + provider.getClass().getName()
- + " as the ClientProtocolProvider - returned null protocol");
- }
- }
- catch (Exception e) {
- LOG.info("Failed to use " + provider.getClass().getName()
- + " due to error: ", e);
+ if (clientProtocol != null) {
+ clientProtocolProvider = provider;
+ client = clientProtocol;
+ LOG.debug("Picked " + provider.getClass().getName()
+ + " as the ClientProtocolProvider");
+ break;
+ } else {
+ LOG.debug("Cannot pick " + provider.getClass().getName()
+ + " as the ClientProtocolProvider - returned null protocol");
}
+ } catch (Exception e) {
+ LOG.info("Failed to use " + provider.getClass().getName()
+ + " due to error: ", e);
}
}