You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2014/02/27 20:25:18 UTC
svn commit: r1572713 - in
/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main...
Author: vinodkv
Date: Thu Feb 27 19:25:18 2014
New Revision: 1572713
URL: http://svn.apache.org/r1572713
Log:
MAPREDUCE-5770. Fixed MapReduce ApplicationMaster to correctly redirect to the YARN's web-app proxy with the correct scheme prefix. Contributed by Jian He.
svn merge --ignore-ancestry -c 1572711 ../../trunk/
Modified:
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt?rev=1572713&r1=1572712&r2=1572713&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/CHANGES.txt Thu Feb 27 19:25:18 2014
@@ -35,6 +35,10 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5757. ConcurrentModificationException in JobControl.toList
(jlowe)
+ MAPREDUCE-5770. Fixed MapReduce ApplicationMaster to correctly redirect
+ to the YARN's web-app proxy with the correct scheme prefix. (Jian He via
+ vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1572713&r1=1572712&r2=1572713&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Thu Feb 27 19:25:18 2014
@@ -1368,13 +1368,7 @@ public class MRAppMaster extends Composi
JobConf conf = new JobConf(new YarnConfiguration());
conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
- // Explicitly disabling SSL for map reduce task as we can't allow MR users
- // to gain access to keystore file for opening SSL listener. We can trust
- // RM/NM to issue SSL certificates but definitely not MR-AM as it is
- // running in user-land.
MRWebAppUtil.initialize(conf);
- conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
- HttpConfig.Policy.HTTP_ONLY.name());
String jobUserName = System
.getenv(ApplicationConstants.Environment.USER.name());
conf.set(MRJobConfig.USER_NAME, jobUserName);
@@ -1470,4 +1464,7 @@ public class MRAppMaster extends Composi
LogManager.shutdown();
}
+ public ClientService getClientService() {
+ return clientService;
+ }
}
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1572713&r1=1572712&r2=1572713&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Thu Feb 27 19:25:18 2014
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -133,8 +134,13 @@ public class MRClientService extends Abs
this.bindAddress = NetUtils.getConnectAddress(server);
LOG.info("Instantiated MRClientService at " + this.bindAddress);
try {
- webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws").with(conf).
- start(new AMWebApp());
+ // Explicitly disabling SSL for map reduce task as we can't allow MR users
+ // to gain access to keystore file for opening SSL listener. We can trust
+ // RM/NM to issue SSL certificates but definitely not MR-AM as it is
+ // running in user-land.
+ webApp =
+ WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
+ .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
} catch (Exception e) {
LOG.error("Webapps failed to start. Ignoring for now:", e);
}
@@ -412,4 +418,8 @@ public class MRClientService extends Abs
" token");
}
}
+
+ public WebApp getWebApp() {
+ return webApp;
+ }
}
Modified: hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java?rev=1572713&r1=1572712&r2=1572713&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java Thu Feb 27 19:25:18 2014
@@ -21,23 +21,45 @@ package org.apache.hadoop.mapreduce.v2.a
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.APP_ID;
import static org.junit.Assert.assertEquals;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
+import javax.net.ssl.SSLException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig.Policy;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.MRApp;
import org.apache.hadoop.mapreduce.v2.app.MockAppContext;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.client.MRClientService;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.Test;
+import com.google.common.net.HttpHeaders;
import com.google.inject.Injector;
public class TestAMWebApp {
@@ -147,7 +169,96 @@ public class TestAMWebApp {
WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
appContext, params);
}
-
+
+ @Test
+ public void testMRWebAppSSLDisabled() throws Exception {
+ MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+ @Override
+ protected ClientService createClientService(AppContext context) {
+ return new MRClientService(context);
+ }
+ };
+ Configuration conf = new Configuration();
+ // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
+ conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
+ Job job = app.submit(conf);
+
+ String hostPort =
+ NetUtils.getHostPortString(((MRClientService) app.getClientService())
+ .getWebApp().getListenerAddress());
+ // http:// should be accessible
+ URL httpUrl = new URL("http://" + hostPort);
+ HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
+ InputStream in = conn.getInputStream();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(in, out, 1024);
+ Assert.assertTrue(out.toString().contains("MapReduce Application"));
+
+ // https:// is not accessible.
+ URL httpsUrl = new URL("https://" + hostPort);
+ try {
+ HttpURLConnection httpsConn =
+ (HttpURLConnection) httpsUrl.openConnection();
+ httpsConn.getInputStream();
+ Assert.fail("https:// is not accessible, expected to fail");
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof SSLException);
+ }
+
+ app.waitForState(job, JobState.SUCCEEDED);
+ app.verifyCompleted();
+ }
+
+ static String webProxyBase = null;
+ public static class TestAMFilterInitializer extends AmFilterInitializer {
+
+ @Override
+ protected String getApplicationWebProxyBase() {
+ return webProxyBase;
+ }
+ }
+
+ @Test
+ public void testMRWebAppRedirection() throws Exception {
+
+ String[] schemePrefix =
+ { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
+ for (String scheme : schemePrefix) {
+ MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+ @Override
+ protected ClientService createClientService(AppContext context) {
+ return new MRClientService(context);
+ }
+ };
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
+ conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme
+ .equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name()
+ : Policy.HTTP_ONLY.name());
+ webProxyBase = "/proxy/" + app.getAppID();
+ conf.set("hadoop.http.filter.initializers",
+ TestAMFilterInitializer.class.getName());
+ Job job = app.submit(conf);
+ String hostPort =
+ NetUtils.getHostPortString(((MRClientService) app.getClientService())
+ .getWebApp().getListenerAddress());
+ URL httpUrl = new URL("http://" + hostPort + "/mapreduce");
+
+ HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
+ conn.setInstanceFollowRedirects(false);
+ conn.connect();
+ String expectedURL =
+ scheme + conf.get(YarnConfiguration.PROXY_ADDRESS)
+ + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce");
+ Assert.assertEquals(expectedURL,
+ conn.getHeaderField(HttpHeaders.LOCATION));
+ Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY,
+ conn.getResponseCode());
+ app.waitForState(job, JobState.SUCCEEDED);
+ app.verifyCompleted();
+ }
+ }
+
public static void main(String[] args) {
WebApps.$for("yarn", AppContext.class, new MockAppContext(0, 8, 88, 4)).
at(58888).inDevMode().start(new AMWebApp()).joinThread();