You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/02/01 17:05:37 UTC
hadoop git commit: MAPREDUCE-6618. YarnClientProtocolProvider leaking
the YarnClient thread. Contributed by Xuan Gong
Repository: hadoop
Updated Branches:
refs/heads/trunk af2dccbca -> 59a212b6e
MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. Contributed by Xuan Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59a212b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59a212b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59a212b6
Branch: refs/heads/trunk
Commit: 59a212b6e1265adfa9b55c71b65a22157dfccf77
Parents: af2dccb
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Feb 1 16:05:06 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Feb 1 16:05:06 2016 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 6 +++++
.../org/apache/hadoop/mapred/ClientCache.java | 24 +++++++++++++++++++-
.../hadoop/mapred/ClientServiceDelegate.java | 16 +++++++++++++
.../org/apache/hadoop/mapred/YARNRunner.java | 11 +++++++++
.../mapred/YarnClientProtocolProvider.java | 5 ++--
.../TestYarnClientProtocolProvider.java | 6 +++--
6 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 8261b34..55284da 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -736,6 +736,9 @@ Release 2.7.3 - UNRELEASED
MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in
MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe)
+ MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
+ (Xuan Gong via jlowe)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
@@ -1041,6 +1044,9 @@ Release 2.6.4 - UNRELEASED
MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in
MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe)
+ MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread.
+ (Xuan Gong via jlowe)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
index 4335c82..93ea5c4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
@@ -22,11 +22,11 @@ import java.io.IOException;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
@@ -97,4 +97,26 @@ public class ClientCache {
}
});
}
+
+ public void close() throws IOException {
+ if (rm != null) {
+ rm.close();
+ }
+
+ if (hsProxy != null) {
+ RPC.stopProxy(hsProxy);
+ hsProxy = null;
+ }
+
+ if (cache != null && !cache.isEmpty()) {
+ for (ClientServiceDelegate delegate : cache.values()) {
+ if (delegate != null) {
+ delegate.close();
+ delegate = null;
+ }
+ }
+ cache.clear();
+ cache = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index 8517c19..eac8dbc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -532,4 +533,19 @@ public class ClientServiceDelegate {
throw new IOException("Cannot get log path for a in-progress job");
}
}
+
+ public void close() throws IOException {
+ if (rm != null) {
+ rm.close();
+ }
+
+ if (historyServerProxy != null) {
+ RPC.stopProxy(historyServerProxy);
+ }
+
+ if (realProxy != null) {
+ RPC.stopProxy(realProxy);
+ realProxy = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index fe631de..1342282 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -756,4 +756,15 @@ public class YARNRunner implements ClientProtocol {
envConf + " config settings.");
}
}
+
+ public void close() throws IOException {
+ if (resMgrDelegate != null) {
+ resMgrDelegate.close();
+ resMgrDelegate = null;
+ }
+ if (clientCache != null) {
+ clientCache.close();
+ clientCache = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
index 29d3f66..6583fa2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java
@@ -44,7 +44,8 @@ public class YarnClientProtocolProvider extends ClientProtocolProvider {
@Override
public void close(ClientProtocol clientProtocol) throws IOException {
- // nothing to do
+ if (clientProtocol instanceof YARNRunner) {
+ ((YARNRunner)clientProtocol).close();
+ }
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59a212b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
index aeb20cd..4d84fa9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
@@ -21,10 +21,11 @@ package org.apache.hadoop.mapreduce;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.doNothing;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
@@ -113,6 +114,8 @@ public class TestYarnClientProtocolProvider extends TestCase {
@Override
protected void serviceStart() throws Exception {
assertTrue(this.client instanceof YarnClientImpl);
+ this.client = spy(this.client);
+ doNothing().when(this.client).close();
((YarnClientImpl) this.client).setRMClient(cRMProtocol);
}
};
@@ -126,5 +129,4 @@ public class TestYarnClientProtocolProvider extends TestCase {
}
}
}
-
}