You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/07/20 08:50:12 UTC
[4/5] hadoop git commit: YARN-5309. Fix SSLFactory truststore
reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang.
YARN-5309. Fix SSLFactory truststore reloader thread leak in TimelineClientImpl. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b8641a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b8641a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b8641a3
Branch: refs/heads/branch-2.7
Commit: 1b8641a3c3b16d798ecd5956ba92b8911cacab95
Parents: a679899
Author: Varun Vasudev <vv...@apache.org>
Authored: Wed Jul 20 13:06:31 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Wed Jul 20 13:06:31 2016 +0530
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 5 ++
.../client/api/impl/TimelineClientImpl.java | 26 ++++++----
.../client/api/impl/TestTimelineClient.java | 53 ++++++++++++++++++++
3 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b8641a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 7373f32..c9eca6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -143,6 +143,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk16</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey.jersey-test-framework</groupId>
<artifactId>jersey-test-framework-grizzly2</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b8641a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 04c84ca..9d85d0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -111,6 +111,7 @@ public class TimelineClientImpl extends TimelineClient {
private URI resURI;
private UserGroupInformation authUgi;
private String doAsUser;
+ private SSLFactory sslFactory;
@Private
@VisibleForTesting
@@ -266,7 +267,7 @@ public class TimelineClientImpl extends TimelineClient {
}
ClientConfig cc = new DefaultClientConfig();
cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
- connConfigurator = newConnConfigurator(conf);
+ connConfigurator = initConnConfigurator(conf);
if (UserGroupInformation.isSecurityEnabled()) {
authenticator = new KerberosDelegationTokenAuthenticator();
} else {
@@ -298,6 +299,14 @@ public class TimelineClientImpl extends TimelineClient {
}
@Override
+ protected void serviceStop() throws Exception {
+ if (this.sslFactory != null) {
+ this.sslFactory.destroy();
+ }
+ super.serviceStop();
+ }
+
+ @Override
public TimelinePutResponse putEntities(
TimelineEntity... entities) throws IOException, YarnException {
TimelineEntities entitiesContainer = new TimelineEntities();
@@ -502,9 +511,9 @@ public class TimelineClientImpl extends TimelineClient {
}
- private static ConnectionConfigurator newConnConfigurator(Configuration conf) {
+ private ConnectionConfigurator initConnConfigurator(Configuration conf) {
try {
- return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ return initSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
} catch (Exception e) {
LOG.debug("Cannot load customized ssl related configuration. " +
"Fallback to system-generic settings.", e);
@@ -522,16 +531,15 @@ public class TimelineClientImpl extends TimelineClient {
}
};
- private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ private ConnectionConfigurator initSslConnConfigurator(final int timeout,
Configuration conf) throws IOException, GeneralSecurityException {
- final SSLFactory factory;
final SSLSocketFactory sf;
final HostnameVerifier hv;
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
+ sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ sslFactory.init();
+ sf = sslFactory.createSSLSocketFactory();
+ hv = sslFactory.getHostnameVerifier();
return new ConnectionConfigurator() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b8641a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index 859a6c9..76f7958 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -25,11 +25,14 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.File;
import java.net.ConnectException;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
@@ -52,9 +55,15 @@ import com.sun.jersey.api.client.ClientResponse;
public class TestTimelineClient {
private TimelineClientImpl client;
+ private static final File testDir = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")),
+ "TestTimelineClient");
@Before
public void setup() {
+ FileUtil.fullyDelete(testDir);
+ testDir.mkdirs();
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
client = createTimelineClient(conf);
@@ -62,6 +71,7 @@ public class TestTimelineClient {
@After
public void tearDown() {
+ FileUtil.fullyDelete(testDir);
if (client != null) {
client.stop();
}
@@ -365,6 +375,49 @@ public class TestTimelineClient {
return client;
}
+ @Test
+ public void testTimelineClientCleanup() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 0);
+
+ String sslConfDir =
+ KeyStoreTestUtil.getClasspathDir(TestTimelineClient.class);
+ KeyStoreTestUtil.setupSSLConfig(testDir.getAbsolutePath(),
+ sslConfDir, conf, false);
+ client = createTimelineClient(conf);
+
+ ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
+
+ while (threadGroup.getParent() != null) {
+ threadGroup = threadGroup.getParent();
+ }
+
+ Thread[] threads = new Thread[threadGroup.activeCount()];
+
+ threadGroup.enumerate(threads);
+ Thread reloaderThread = null;
+ for (Thread thread : threads) {
+ if ((thread.getName() != null)
+ && (thread.getName().contains("Truststore reloader thread"))) {
+ reloaderThread = thread;
+ }
+ }
+ Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive());
+
+ client.close();
+
+ boolean reloaderStillAlive = true;
+ for (int i = 0; i < 10; i++) {
+ reloaderStillAlive = reloaderThread.isAlive();
+ if (!reloaderStillAlive) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ Assert.assertFalse("Reloader is still alive", reloaderStillAlive);
+ }
+
private static class TestTimlineDelegationTokenSecretManager extends
AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org