You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/01/09 16:34:23 UTC
[02/10] incubator-slider git commit: SLIDER-710 restore AM filter
(with an XML conf key to enable it)
SLIDER-710 restore AM filter (with an XML conf key to enable it)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e5fb7f83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e5fb7f83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e5fb7f83
Branch: refs/heads/develop
Commit: e5fb7f83ea1e7d45020aaa962bfe2518672450db
Parents: ceb21e1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jan 7 14:19:56 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jan 7 14:19:56 2015 +0000
----------------------------------------------------------------------
.../common/SliderXMLConfKeysForTesting.java | 9 +-
.../apache/slider/common/SliderXmlConfKeys.java | 5 +
.../server/appmaster/SliderAppMaster.java | 104 ++++++++++++-------
.../appmaster/web/rest/InsecureAmFilter.java | 101 ++++++++++++++++++
.../web/rest/InsecureAmFilterInitializer.java | 103 ++++++++++++++++++
5 files changed, 278 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e5fb7f83/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index 41c61d4..0a7f292 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -22,14 +22,7 @@ package org.apache.slider.common;
* Keys shared across tests
*/
public interface SliderXMLConfKeysForTesting {
-
- String KEY_TEST_HBASE_HOME = "slider.test.hbase.home";
- String KEY_TEST_HBASE_TAR = "slider.test.hbase.tar";
- String KEY_TEST_HBASE_APPCONF = "slider.test.hbase.appconf";
- String KEY_TEST_ACCUMULO_HOME = "slider.test.accumulo.home";
- String KEY_TEST_ACCUMULO_TAR = "slider.test.accumulo.tar";
- String KEY_TEST_ACCUMULO_APPCONF = "slider.test.accumulo.appconf";
-
+
String KEY_TEST_THAW_WAIT_TIME = "slider.test.thaw.wait.seconds";
int DEFAULT_THAW_WAIT_TIME_SECONDS = 60;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e5fb7f83/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 9b1316e..0672955 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -143,4 +143,9 @@ public interface SliderXmlConfKeys {
* The path to the python executable utilized to launch the agent.
*/
String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
+
+ /**
+ * Flag to enable the insecure AM filter: {@value}
+ */
+ String X_DEV_INSECURE_WS = "slider.dev.ws.insecure";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e5fb7f83/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7b9f6db..c34c692 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -146,6 +146,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.AgentService;
+import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
import org.apache.slider.server.appmaster.web.WebAppApi;
@@ -385,6 +386,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* The port for the web application
*/
private int webAppPort;
+ private boolean securityEnabled;
/**
* Service Constructor
@@ -593,7 +595,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
securityConfiguration = new SecurityConfiguration(
serviceConf, instanceDefinition, clustername);
// obtain security state
- boolean securityEnabled = securityConfiguration.isSecurityEnabled();
+ securityEnabled = securityConfiguration.isSecurityEnabled();
// set the global security flag for the instance definition
instanceDefinition.getAppConfOperations().set(
KEY_SECURITY_ENABLED, securityEnabled);
@@ -607,9 +609,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
File parentFile = confDir.getParentFile();
log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile));
}
-
- // IP filtering
- serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME);
//get our provider
MapOperations globalInternalOptions = getGlobalInternalOptions();
@@ -655,6 +654,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Map<String, String> envVars;
List<Container> liveContainers;
+
/**
* It is critical this section is synchronized, to stop async AM events
* arriving while registering a restarting AM.
@@ -719,7 +719,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
uploadServerCertForLocalization(clustername, fs);
}
- startAgentWebApp(appInformation, serviceConf);
webAppPort = getPortToRequest();
if (webAppPort == 0) {
@@ -795,7 +794,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
providerService.validateApplicationConfiguration(instanceDefinition,
confDir,
- securityEnabled);
+ securityEnabled);
//determine the location for the role history data
Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME);
@@ -815,11 +814,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
instanceDefinition.getName(), appState.getRolePriorityMap());
// add the AM to the list of nodes in the cluster
-
+
appState.buildAppMasterNode(appMasterContainerID,
- appMasterHostname,
+ appMasterHostname,
webAppPort,
- appMasterHostname + ":" + webAppPort);
+ appMasterHostname + ":" + webAppPort);
// build up environment variables that the AM wants set in every container
// irrespective of provider and role.
@@ -872,13 +871,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
service_user_name = RegistryUtils.currentUser();
log.info("Registry service username ={}", service_user_name);
- // now do the registration
- registerServiceInstance(clustername, appid);
- // log the YARN and web UIs
- log.info("RM Webapp address {}", serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
- log.info("slider Webapp address {}", appMasterTrackingUrl);
-
// declare the cluster initialized
log.info("Application Master Initialization Completed");
initCompleted.set(true);
@@ -891,8 +884,28 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
startQueueProcessing();
- deployWebApplication(serviceConf, webAppPort);
+ // Web service endpoints: initialize
+
+ WebAppApiImpl webAppApi =
+ new WebAppApiImpl(this,
+ stateForProviders,
+ providerService,
+ certificateManager,
+ registryOperations,
+ metricsAndMonitoring);
+ initAMFilterOptions(serviceConf);
+ // start the agent web app
+ startAgentWebApp(appInformation, serviceConf, webAppApi);
+ deployWebApplication(serviceConf, webAppPort, webAppApi);
+
+ // YARN Registry do the registration
+ registerServiceInstance(clustername, appid);
+
+ // log the YARN and web UIs
+ log.info("RM Webapp address {}",
+ serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
+ log.info("slider Webapp address {}", appMasterTrackingUrl);
// Start the Slider AM provider
sliderAMProvider.start();
@@ -917,17 +930,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Creates and starts the web application, and adds a
* <code>WebAppService</code> service under the AM, to ensure
* a managed web application shutdown.
- *
- * @param serviceConf AM configuration
+ * @param serviceConf AM configuration
* @param port port to deploy the web application on
+ * @param webAppApi web app API instance
*/
- private void deployWebApplication(Configuration serviceConf, int port) {
- WebAppApi webAppApi = new WebAppApiImpl(this,
- stateForProviders,
- providerService,
- certificateManager,
- registryOperations,
- metricsAndMonitoring);
+ private void deployWebApplication(Configuration serviceConf,
+ int port, WebAppApiImpl webAppApi) {
+
webApp = new SliderAMWebApp(webAppApi);
WebApps.$for(SliderAMWebApp.BASE_PATH,
WebAppApi.class,
@@ -1061,26 +1070,31 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
+ /**
+ * Set up and start the agent web application
+ * @param appInformation application information
+ * @param serviceConf service configuration
+ * @param webAppApi web app API instance to bind to
+ * @throws IOException
+ */
private void startAgentWebApp(MapOperations appInformation,
- Configuration serviceConf) throws IOException {
+ Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
StringBuilder sb = new StringBuilder("AM classpath:");
for (URL url : urls) {
sb.append("\n").append(url.toString());
}
- LOG_YARN.info(sb.append("\n").toString());
+ LOG_YARN.debug(sb.append("\n").toString());
+ initAMFilterOptions(serviceConf);
+
+
// Start up the agent web app and track the URL for it
+ MapOperations appMasterConfig = getInstanceDefinition()
+ .getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
AgentWebApp agentWebApp = AgentWebApp.$for(AgentWebApp.BASE_PATH,
- new WebAppApiImpl(this,
- stateForProviders,
- providerService,
- certificateManager,
- registryOperations,
- metricsAndMonitoring),
+ webAppApi,
RestPaths.AGENT_WS_CONTEXT)
- .withComponentConfig(getInstanceDefinition().getAppConfOperations()
- .getComponent(
- SliderKeys.COMPONENT_AM))
+ .withComponentConfig(appMasterConfig)
.start();
agentOpsUrl =
"https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
@@ -1102,6 +1116,24 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
+ * Set up the AM filter
+ * @param serviceConf configuration to patch
+ */
+ private void initAMFilterOptions(Configuration serviceConf) {
+ // IP filtering
+ String amFilterName = AM_FILTER_NAME;
+
+ // This is here until YARN supports proxy & redirect operations
+ // on verbs other than GET, and is only supported for testing
+ if (serviceConf.getBoolean(SliderXmlConfKeys.X_DEV_INSECURE_WS, false)) {
+ log.warn("Insecure filter enabled: REST operations are unauthenticated");
+ amFilterName = InsecureAmFilterInitializer.NAME;
+ }
+
+ serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, amFilterName);
+ }
+
+ /**
* This registers the service instance and its external values
* @param instanceName name of this instance
* @param appid application ID
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e5fb7f83/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
new file mode 100644
index 0000000..07b19e7
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest;
+
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpPrincipal;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpServletRequestWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+/**
+ * This is a filter which is used to forward insecure operations
+ * There's some metrics to track all operations too
+ */
+public class InsecureAmFilter extends AmIpFilter {
+ public static final String WS_CONTEXT_ROOT = "slider.rest.context.root";
+ protected static final Logger log =
+ LoggerFactory.getLogger(InsecureAmFilter.class);
+
+ private String wsContextRoot;
+
+
+ @Override
+ public void init(FilterConfig conf) throws ServletException {
+ super.init(conf);
+ wsContextRoot = conf.getInitParameter(WS_CONTEXT_ROOT);
+ }
+
+ private void rejectNonHttpRequests(ServletRequest req) throws
+ ServletException {
+ if (!(req instanceof HttpServletRequest)) {
+ throw new ServletException("This filter only works for HTTP/HTTPS");
+ }
+ }
+
+ @Override
+ public void doFilter(ServletRequest req,
+ ServletResponse resp,
+ FilterChain chain) throws IOException, ServletException {
+ rejectNonHttpRequests(req);
+ HttpServletRequest httpReq = (HttpServletRequest) req;
+ HttpServletResponse httpResp = (HttpServletResponse) resp;
+
+
+ if (!httpReq.getRequestURI().startsWith(wsContextRoot)) {
+ // hand off to the AM filter if it is not the context root
+ super.doFilter(req, resp, chain);
+ return;
+ }
+
+ String user = null;
+
+ if (httpReq.getCookies() != null) {
+ for (Cookie c : httpReq.getCookies()) {
+ if (WebAppProxyServlet.PROXY_USER_COOKIE_NAME.equals(c.getName())) {
+ user = c.getValue();
+ break;
+ }
+ }
+ }
+
+ if (user == null) {
+ log.warn("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ + " cookie, so user will not be set");
+ chain.doFilter(req, resp);
+ } else {
+ final AmIpPrincipal principal = new AmIpPrincipal(user);
+ ServletRequest requestWrapper = new AmIpServletRequestWrapper(httpReq,
+ principal);
+ chain.doFilter(requestWrapper, resp);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e5fb7f83/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java
new file mode 100644
index 0000000..111d715
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.FilterContainer;
+import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InsecureAmFilterInitializer extends FilterInitializer {
+ private static final String FILTER_NAME = "AM_PROXY_FILTER";
+ private static final String FILTER_CLASS =
+ InsecureAmFilter.class.getCanonicalName();
+ private static final String HTTPS_PREFIX = "https://";
+ private static final String HTTP_PREFIX = "http://";
+
+ static final String PROXY_HOSTS = "PROXY_HOSTS";
+ static final String PROXY_HOSTS_DELIMITER = ",";
+ static final String PROXY_URI_BASES = "PROXY_URI_BASES";
+ static final String PROXY_URI_BASES_DELIMITER = ",";
+
+ private Configuration configuration;
+
+ public static final String NAME =
+ "org.apache.slider.server.appmaster.web.InsecureAmFilterInitializer";
+
+ @Override
+ public void initFilter(FilterContainer container, Configuration conf) {
+ configuration = conf;
+ Map<String, String> params = new HashMap<String, String>();
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ String[] parts = proxy.split(":");
+ params.put(InsecureAmFilter.PROXY_HOST, parts[0]);
+ // todo: eventually call WebAppUtils.getHttpSchemePrefix
+ params.put(InsecureAmFilter.PROXY_URI_BASE, getHttpSchemePrefix()
+ + proxy +
+ getApplicationWebProxyBase());
+ params.put(InsecureAmFilter.WS_CONTEXT_ROOT,
+ conf.get(InsecureAmFilter.WS_CONTEXT_ROOT));
+ container.addFilter(FILTER_NAME, FILTER_CLASS, params);
+ }
+
+ private void classicAmFilterInitializerInit(FilterContainer container,
+ Configuration conf) {
+ Map<String, String> params = new HashMap<String, String>();
+ List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
+ StringBuilder sb = new StringBuilder();
+ for (String proxy : proxies) {
+ sb.append(proxy.split(":")[0]).append(PROXY_HOSTS_DELIMITER);
+ }
+ sb.setLength(sb.length() - 1);
+ params.put(PROXY_HOSTS, sb.toString());
+
+ String prefix = WebAppUtils.getHttpSchemePrefix(conf);
+ String proxyBase = getApplicationWebProxyBase();
+ sb = new StringBuilder();
+ for (String proxy : proxies) {
+ sb.append(prefix).append(proxy).append(proxyBase)
+ .append(PROXY_HOSTS_DELIMITER);
+ }
+ sb.setLength(sb.length() - 1);
+ params.put(PROXY_URI_BASES, sb.toString());
+
+ }
+
+ @VisibleForTesting
+ protected String getApplicationWebProxyBase() {
+ return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
+ }
+
+ private String getHttpSchemePrefix() {
+ return HttpConfig.Policy.HTTPS_ONLY ==
+ HttpConfig.Policy.fromString(configuration
+ .get(
+ YarnConfiguration.YARN_HTTP_POLICY_KEY,
+ YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))
+ ? HTTPS_PREFIX : HTTP_PREFIX;
+ }
+}