You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/10/24 05:53:06 UTC
[46/50] [abbrv] hadoop git commit: MAPREDUCE-4669. MRAM web UI does
not work with HTTPS. (Contributed by Robert Kanter)
MAPREDUCE-4669. MRAM web UI does not work with HTTPS. (Contributed by Robert Kanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/823bb5dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/823bb5dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/823bb5dd
Branch: refs/heads/HDFS-12943
Commit: 823bb5dda88b2a9a1374c2a52af8d0e9b6bb801a
Parents: 635786a
Author: Haibo Chen <ha...@apache.org>
Authored: Tue Oct 23 15:27:37 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Oct 23 15:28:13 2018 -0700
----------------------------------------------------------------------
.../hadoop/security/ssl/KeyStoreTestUtil.java | 105 ++++++++++++
.../hadoop-mapreduce-client-app/pom.xml | 5 +
.../v2/app/client/MRClientService.java | 15 +-
.../mapreduce/v2/app/webapp/TestAMWebApp.java | 158 ++++++++++++++++++-
.../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 6 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 22 +++
.../src/main/resources/mapred-default.xml | 21 +++
hadoop-project/pom.xml | 5 +
.../org/apache/hadoop/yarn/webapp/WebApps.java | 25 ++-
9 files changed, 352 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
index 1870b22..0d30e6e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java
@@ -32,15 +32,20 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.math.BigInteger;
+import java.net.Socket;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.Key;
+import java.security.KeyManagementException;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.KeyStore;
import java.security.NoSuchAlgorithmException;
+import java.security.Principal;
+import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Date;
import java.util.HashMap;
@@ -50,8 +55,15 @@ import java.security.InvalidKeyException;
import java.security.NoSuchProviderException;
import java.security.SignatureException;
import java.security.cert.CertificateEncodingException;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509KeyManager;
+import javax.net.ssl.X509TrustManager;
import javax.security.auth.x500.X500Principal;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.bouncycastle.x509.X509V1CertificateGenerator;
public class KeyStoreTestUtil {
@@ -538,4 +550,97 @@ public class KeyStoreTestUtil {
sslConf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile);
return sslConf;
}
+
+ /**
+ * Configures the passed in {@link HttpsURLConnection} to allow all SSL
+ * certificates.
+ *
+ * @param httpsConn The HttpsURLConnection to configure
+ * @throws KeyManagementException
+ * @throws NoSuchAlgorithmException
+ */
+ public static void setAllowAllSSL(HttpsURLConnection httpsConn)
+ throws KeyManagementException, NoSuchAlgorithmException {
+ setAllowAllSSL(httpsConn, null);
+ }
+
+ /**
+ * Configures the passed in {@link HttpsURLConnection} to allow all SSL
+ * certificates. Also presents a client certificate.
+ *
+ * @param httpsConn The HttpsURLConnection to configure
+ * @param clientCert The client certificate to present
+ * @param clientKeyPair The KeyPair for the client certificate
+ * @throws KeyManagementException
+ * @throws NoSuchAlgorithmException
+ */
+ public static void setAllowAllSSL(HttpsURLConnection httpsConn,
+ X509Certificate clientCert, KeyPair clientKeyPair)
+ throws KeyManagementException, NoSuchAlgorithmException {
+ X509KeyManager km = new X509KeyManager() {
+ @Override
+ public String[] getClientAliases(String s, Principal[] principals) {
+ return new String[]{"client"};
+ }
+
+ @Override
+ public String chooseClientAlias(String[] strings,
+ Principal[] principals, Socket socket) {
+ return "client";
+ }
+
+ @Override
+ public String[] getServerAliases(String s, Principal[] principals) {
+ return null;
+ }
+
+ @Override
+ public String chooseServerAlias(String s, Principal[] principals,
+ Socket socket) {
+ return null;
+ }
+
+ @Override
+ public X509Certificate[] getCertificateChain(String s) {
+ return new X509Certificate[]{clientCert};
+ }
+
+ @Override
+ public PrivateKey getPrivateKey(String s) {
+ return clientKeyPair.getPrivate();
+ }
+ };
+ setAllowAllSSL(httpsConn, km);
+ }
+
+ private static void setAllowAllSSL(HttpsURLConnection httpsConn,
+ KeyManager km) throws KeyManagementException, NoSuchAlgorithmException {
+ // Create a TrustManager that trusts anything
+ TrustManager[] trustAllCerts = new TrustManager[] {
+ new X509TrustManager() {
+ @Override
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[]{};
+ }
+
+ @Override
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] certs, String authType)
+ throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] certs, String authType)
+ throws CertificateException {
+ }
+ }
+ };
+ KeyManager[] kms = (km == null) ? null : new KeyManager[]{km};
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(kms, trustAllCerts, new SecureRandom());
+ httpsConn.setSSLSocketFactory(sc.getSocketFactory());
+ // Don't check the hostname
+ httpsConn.setHostnameVerifier(new NoopHostnameVerifier());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index fab6466..2d7ea98 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -108,6 +108,11 @@
<artifactId>bcpkix-jdk15on</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
index b2dc8ad..328a4b9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
@@ -25,6 +25,7 @@ import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.mapreduce.JobACL;
@@ -136,14 +137,18 @@ public class MRClientService extends AbstractService implements ClientService {
server.getListenerAddress().getPort());
LOG.info("Instantiated MRClientService at " + this.bindAddress);
try {
- // Explicitly disabling SSL for map reduce task as we can't allow MR users
- // to gain access to keystore file for opening SSL listener. We can trust
- // RM/NM to issue SSL certificates but definitely not MR-AM as it is
- // running in user-land.
+ HttpConfig.Policy httpPolicy = conf.getBoolean(
+ MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED,
+ MRJobConfig.DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED)
+ ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY;
+ boolean needsClientAuth = conf.getBoolean(
+ MRJobConfig.MR_AM_WEBAPP_HTTPS_CLIENT_AUTH,
+ MRJobConfig.DEFAULT_MR_AM_WEBAPP_HTTPS_CLIENT_AUTH);
webApp =
WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
- .withHttpPolicy(conf, Policy.HTTP_ONLY)
+ .withHttpPolicy(conf, httpPolicy)
.withPortRange(conf, MRJobConfig.MR_AM_WEBAPP_PORT_RANGE)
+ .needsClientAuth(needsClientAuth)
.start(new AMWebApp());
} catch (Exception e) {
LOG.error("Webapps failed to start. Ignoring for now:", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
index 21d37c8..d55c929 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
@@ -22,15 +22,23 @@ import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.APP_ID;
import static org.junit.Assert.assertEquals;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.InputStream;
import java.net.HttpURLConnection;
+import java.net.SocketException;
import java.net.URL;
+import java.security.KeyPair;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
+import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLException;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -57,13 +65,26 @@ import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.http.HttpStatus;
+import org.junit.After;
+import org.junit.Rule;
import org.junit.Test;
import com.google.common.net.HttpHeaders;
import com.google.inject.Injector;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
public class TestAMWebApp {
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")),
+ TestAMWebApp.class.getName());
+
+ @After
+ public void tearDown() {
+ TEST_DIR.delete();
+ }
+
@Test public void testAppControllerIndex() {
AppContext ctx = new MockAppContext(0, 1, 1, 1);
Injector injector = WebAppTests.createMockInjector(AppContext.class, ctx);
@@ -179,7 +200,7 @@ public class TestAMWebApp {
}
};
Configuration conf = new Configuration();
- // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
+ // MR is explicitly disabling SSL, even though YARN setting as HTTPS_ONLY
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
Job job = app.submit(conf);
@@ -201,12 +222,143 @@ public class TestAMWebApp {
(HttpURLConnection) httpsUrl.openConnection();
httpsConn.getInputStream();
Assert.fail("https:// is not accessible, expected to fail");
- } catch (Exception e) {
- Assert.assertTrue(e instanceof SSLException);
+ } catch (SSLException e) {
+ // expected
+ }
+
+ app.waitForState(job, JobState.SUCCEEDED);
+ app.verifyCompleted();
+ }
+
+ @Rule
+ public final EnvironmentVariables environmentVariables
+ = new EnvironmentVariables();
+
+ @Test
+ public void testMRWebAppSSLEnabled() throws Exception {
+ MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+ @Override
+ protected ClientService createClientService(AppContext context) {
+ return new MRClientService(context);
+ }
+ };
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED, true);
+
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ Certificate cert = KeyStoreTestUtil.generateCertificate(
+ "CN=foo", keyPair, 5, "SHA512WITHRSA");
+ File keystoreFile = new File(TEST_DIR, "server.keystore");
+ keystoreFile.getParentFile().mkdirs();
+ KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), "password",
+ "server", keyPair.getPrivate(), cert);
+ environmentVariables.set("KEYSTORE_FILE_LOCATION",
+ keystoreFile.getAbsolutePath());
+ environmentVariables.set("KEYSTORE_PASSWORD", "password");
+
+ Job job = app.submit(conf);
+
+ String hostPort =
+ NetUtils.getHostPortString(((MRClientService) app.getClientService())
+ .getWebApp().getListenerAddress());
+ // https:// should be accessible
+ URL httpsUrl = new URL("https://" + hostPort);
+ HttpsURLConnection httpsConn =
+ (HttpsURLConnection) httpsUrl.openConnection();
+ KeyStoreTestUtil.setAllowAllSSL(httpsConn);
+
+ InputStream in = httpsConn.getInputStream();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(in, out, 1024);
+ Assert.assertTrue(out.toString().contains("MapReduce Application"));
+
+ // http:// is not accessible.
+ URL httpUrl = new URL("http://" + hostPort);
+ try {
+ HttpURLConnection httpConn =
+ (HttpURLConnection) httpUrl.openConnection();
+ httpConn.getResponseCode();
+ Assert.fail("http:// is not accessible, expected to fail");
+ } catch (SocketException e) {
+ // expected
}
app.waitForState(job, JobState.SUCCEEDED);
app.verifyCompleted();
+
+ keystoreFile.delete();
+ }
+
+ @Test
+ public void testMRWebAppSSLEnabledWithClientAuth() throws Exception {
+ MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+ @Override
+ protected ClientService createClientService(AppContext context) {
+ return new MRClientService(context);
+ }
+ };
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED, true);
+ conf.setBoolean(MRJobConfig.MR_AM_WEBAPP_HTTPS_CLIENT_AUTH, true);
+
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ Certificate cert = KeyStoreTestUtil.generateCertificate(
+ "CN=foo", keyPair, 5, "SHA512WITHRSA");
+ File keystoreFile = new File(TEST_DIR, "server.keystore");
+ keystoreFile.getParentFile().mkdirs();
+ KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), "password",
+ "server", keyPair.getPrivate(), cert);
+ environmentVariables.set("KEYSTORE_FILE_LOCATION",
+ keystoreFile.getAbsolutePath());
+ environmentVariables.set("KEYSTORE_PASSWORD", "password");
+
+ KeyPair clientKeyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ X509Certificate clientCert = KeyStoreTestUtil.generateCertificate(
+ "CN=bar", clientKeyPair, 5, "SHA512WITHRSA");
+ File truststoreFile = new File(TEST_DIR, "client.truststore");
+ truststoreFile.getParentFile().mkdirs();
+ KeyStoreTestUtil.createTrustStore(truststoreFile.getAbsolutePath(),
+ "password", "client", clientCert);
+ environmentVariables.set("TRUSTSTORE_FILE_LOCATION",
+ truststoreFile.getAbsolutePath());
+ environmentVariables.set("TRUSTSTORE_PASSWORD", "password");
+
+ Job job = app.submit(conf);
+
+ String hostPort =
+ NetUtils.getHostPortString(((MRClientService) app.getClientService())
+ .getWebApp().getListenerAddress());
+ // https:// should be accessible
+ URL httpsUrl = new URL("https://" + hostPort);
+ HttpsURLConnection httpsConn =
+ (HttpsURLConnection) httpsUrl.openConnection();
+ KeyStoreTestUtil.setAllowAllSSL(httpsConn, clientCert, clientKeyPair);
+
+ InputStream in = httpsConn.getInputStream();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(in, out, 1024);
+ Assert.assertTrue(out.toString().contains("MapReduce Application"));
+
+ // Try with wrong client cert
+ KeyPair otherClientKeyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ X509Certificate otherClientCert = KeyStoreTestUtil.generateCertificate(
+ "CN=bar", otherClientKeyPair, 5, "SHA512WITHRSA");
+ KeyStoreTestUtil.setAllowAllSSL(httpsConn, otherClientCert, clientKeyPair);
+
+ try {
+ HttpURLConnection httpConn =
+ (HttpURLConnection) httpsUrl.openConnection();
+ httpConn.getResponseCode();
+ Assert.fail("Wrong client certificate, expected to fail");
+ } catch (SSLException e) {
+ // expected
+ }
+
+ app.waitForState(job, JobState.SUCCEEDED);
+ app.verifyCompleted();
+
+ keystoreFile.delete();
+ truststoreFile.delete();
}
static String webProxyBase = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
index 6f2e21f..8a6e138 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.net.NetUtils;
@@ -178,6 +179,9 @@ public class MRWebAppUtil {
}
public static String getAMWebappScheme(Configuration conf) {
- return "http://";
+ return conf.getBoolean(
+ MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED,
+ MRJobConfig.DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED)
+ ? "https://" : "http://";
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index ca18bfe..3592b3d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -762,6 +762,28 @@ public interface MRJobConfig {
*/
String MR_AM_WEBAPP_PORT_RANGE = MR_AM_PREFIX + "webapp.port-range";
+ /**
+ * True if the MR AM should use HTTPS for its webapp. If
+ * {@link org.apache.hadoop.yarn.conf.YarnConfiguration#RM_APPLICATION_HTTPS_POLICY}
+ * is set to LENIENT or STRICT, the MR AM will automatically use the
+ * keystore provided by YARN with a certificate for the MR AM webapp, unless
+ * provided by the user.
+ */
+ String MR_AM_WEBAPP_HTTPS_ENABLED = MR_AM_PREFIX + "webapp.https.enabled";
+ boolean DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED = false;
+
+ /**
+ * True if the MR AM webapp should require client HTTPS authentication (i.e.
+ * the proxy server (RM) should present a certificate to the MR AM webapp).
+ * If {@link org.apache.hadoop.yarn.conf.YarnConfiguration#RM_APPLICATION_HTTPS_POLICY}
+ * is set to LENIENT or STRICT, the MR AM will automatically use the
+ * truststore provided by YARN with the RMs certificate, unless provided by
+ * the user.
+ */
+ String MR_AM_WEBAPP_HTTPS_CLIENT_AUTH =
+ MR_AM_PREFIX + "webapp.https.client.auth";
+ boolean DEFAULT_MR_AM_WEBAPP_HTTPS_CLIENT_AUTH = false;
+
/** Enable blacklisting of nodes in the job.*/
public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE =
MR_AM_PREFIX + "job.node-blacklisting.enable";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index e5da41f..ccc9c3d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1482,6 +1482,27 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.webapp.https.enabled</name>
+ <value>false</value>
+ <description>True if the MR AM should use HTTPS for its webapp. If
+ yarn.resourcemanager.application-https.policy is set to LENIENT or STRICT,
+ the MR AM will automatically use the keystore provided by YARN with a
+ certificate for the MR AM webapp, unless provided by the user.
+ </description>
+</property>
+
+<property>
+ <name>yarn.app.mapreduce.am.webapp.https.client.auth</name>
+ <value>false</value>
+ <description>True if the MR AM webapp should require client HTTPS
+ authentication (i.e. the proxy server (RM) should present a certificate to
+ the MR AM webapp). If yarn.resourcemanager.application-https.policy is set
+ to LENIENT or STRICT, the MR AM will automatically use the truststore
+ provided by YARN with the RMs certificate, unless provided by the user.
+ </description>
+</property>
+
+<property>
<name>yarn.app.mapreduce.am.job.committer.cancel-timeout</name>
<value>60000</value>
<description>The amount of time in milliseconds to wait for the output
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e5f4965..3a44e8c 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1074,6 +1074,11 @@
<version>4.11</version>
</dependency>
<dependency>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ <version>1.18.0</version>
+ </dependency>
+ <dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/823bb5dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 0e9f0a7..ced5ced 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -95,6 +95,7 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
Policy httpPolicy = null;
+ boolean needsClientAuth = false;
String portRangeConfigKey = null;
boolean devMode = false;
private String spnegoPrincipalKey;
@@ -174,6 +175,11 @@ public class WebApps {
return this;
}
+ public Builder<T> needsClientAuth(boolean needsClientAuth) {
+ this.needsClientAuth = needsClientAuth;
+ return this;
+ }
+
/**
* Set port range config key and associated configuration object.
* @param config configuration.
@@ -335,7 +341,24 @@ public class WebApps {
}
if (httpScheme.equals(WebAppUtils.HTTPS_PREFIX)) {
- WebAppUtils.loadSslConfiguration(builder, conf);
+ String amKeystoreLoc = System.getenv("KEYSTORE_FILE_LOCATION");
+ if (amKeystoreLoc != null) {
+ LOG.info("Setting keystore location to " + amKeystoreLoc);
+ String password = System.getenv("KEYSTORE_PASSWORD");
+ builder.keyStore(amKeystoreLoc, password, "jks");
+ } else {
+ LOG.info("Loading standard ssl config");
+ WebAppUtils.loadSslConfiguration(builder, conf);
+ }
+ builder.needsClientAuth(needsClientAuth);
+ if (needsClientAuth) {
+ String amTruststoreLoc = System.getenv("TRUSTSTORE_FILE_LOCATION");
+ if (amTruststoreLoc != null) {
+ LOG.info("Setting truststore location to " + amTruststoreLoc);
+ String password = System.getenv("TRUSTSTORE_PASSWORD");
+ builder.trustStore(amTruststoreLoc, password, "jks");
+ }
+ }
}
HttpServer2 server = builder.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org