You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2022/10/11 17:19:07 UTC
[hadoop] branch trunk updated: YARN-11315. [Federation] YARN Federation Router Supports Cross-Origin. (#4934)
This is an automated email from the ASF dual-hosted git repository.
inigoiri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 82a88a8ae62 YARN-11315. [Federation] YARN Federation Router Supports Cross-Origin. (#4934)
82a88a8ae62 is described below
commit 82a88a8ae626a24ffe6502021a76682ce97b9416
Author: slfan1989 <55...@users.noreply.github.com>
AuthorDate: Wed Oct 12 01:18:50 2022 +0800
YARN-11315. [Federation] YARN Federation Router Supports Cross-Origin. (#4934)
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../java/org/apache/hadoop/yarn/webapp/WebApp.java | 5 +
.../src/main/resources/yarn-default.xml | 13 ++-
.../hadoop-yarn-server-router/pom.xml | 6 ++
.../apache/hadoop/yarn/server/router/Router.java | 11 +++
.../hadoop/yarn/server/router/TestRouter.java | 109 +++++++++++++++++++++
.../src/site/markdown/Federation.md | 10 ++
.../hadoop-yarn-site/src/site/markdown/YarnUI2.md | 1 +
8 files changed, 159 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d5e120695e7..7427533fbe7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -4194,6 +4194,11 @@ public class YarnConfiguration extends Configuration {
ROUTER_WEBAPP_PREFIX + "appsinfo-cached-count";
public static final int DEFAULT_ROUTER_APPSINFO_CACHED_COUNT = 100;
+ /** Enable cross origin (CORS) support. **/
+ public static final String ROUTER_WEBAPP_ENABLE_CORS_FILTER =
+ ROUTER_PREFIX + "webapp.cross-origin.enabled";
+ public static final boolean DEFAULT_ROUTER_WEBAPP_ENABLE_CORS_FILTER = false;
+
////////////////////////////////
// CSI Volume configs
////////////////////////////////
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
index 4898c266777..6ef1c50cc6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.util.Lists;
@@ -299,4 +300,8 @@ public abstract class WebApp extends ServletModule {
public abstract void setup();
+ @VisibleForTesting
+ public HttpServer2 getHttpServer() {
+ return httpServer;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5132d4199af..fdfa1296dc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -5006,7 +5006,18 @@
Default is 100
</description>
</property>
-
+
+ <property>
+ <name>yarn.router.webapp.cross-origin.enabled</name>
+ <value>false</value>
+ <description>
+ Flag to enable cross-origin (CORS) support for Yarn Router.
+ For Yarn Router, also add
+ org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the
+ configuration hadoop.http.filter.initializers in core-site.xml.
+ </description>
+ </property>
+
<property>
<name>yarn.federation.state-store.max-applications</name>
<value>1000</value>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
index aa808801df4..cd8a6dfdb9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
@@ -129,6 +129,12 @@
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.grizzly</groupId>
+ <artifactId>grizzly-http-servlet</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
index e95b25678bf..82a1e1ea6f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.JvmPauseMonitor;
@@ -168,6 +169,16 @@ public class Router extends CompositeService {
@VisibleForTesting
public void startWepApp() {
+ // Initialize RouterWeb's CrossOrigin capability.
+ boolean enableCors = conf.getBoolean(YarnConfiguration.ROUTER_WEBAPP_ENABLE_CORS_FILTER,
+ YarnConfiguration.DEFAULT_ROUTER_WEBAPP_ENABLE_CORS_FILTER);
+ if (enableCors) {
+ conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+ + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
+ }
+
+ LOG.info("Instantiating RouterWebApp at {}.", webAppAddress);
+
RMWebAppUtil.setupSecurityAndFilters(conf, null);
Builder<Object> builder =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
index 9f0b4c72aac..d5501d74440 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
@@ -22,11 +22,27 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.hadoop.security.http.CrossOriginFilter;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
/**
* Tests {@link Router}.
@@ -87,4 +103,97 @@ public class TestRouter {
}
}
+ @Test
+ public void testRouterSupportCrossOrigin() throws ServletException, IOException {
+
+ // We design test cases like this
+ // We start the Router and enable the Router to support Cross-origin.
+ // In the configuration, we allow example.com to access.
+ // 1. We simulate example.com and get the correct response
+ // 2. We simulate example.org and cannot get a response
+
+ // Initialize RouterWeb's CrossOrigin capability
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.ROUTER_WEBAPP_ENABLE_CORS_FILTER, true);
+ conf.set("hadoop.http.filter.initializers", HttpCrossOriginFilterInitializer.class.getName());
+ conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_ORIGINS,
+ "example.com");
+ conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_HEADERS,
+ "X-Requested-With,Accept");
+ conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_METHODS,
+ "GET,POST");
+
+ // Start the router
+ Router router = new Router();
+ router.init(conf);
+ router.start();
+ router.getServices();
+
+ // Get assigned to Filter.
+ // The name of the filter is "Cross Origin Filter",
+ // which is specified in HttpCrossOriginFilterInitializer.
+ WebApp webApp = router.getWebapp();
+ HttpServer2 httpServer2 = webApp.getHttpServer();
+ WebAppContext webAppContext = httpServer2.getWebAppContext();
+ ServletHandler servletHandler = webAppContext.getServletHandler();
+ FilterHolder holder = servletHandler.getFilter("Cross Origin Filter");
+ CrossOriginFilter filter = CrossOriginFilter.class.cast(holder.getFilter());
+
+ // 1. Simulate [example.com] for access
+ HttpServletRequest mockReq = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(mockReq.getHeader("Origin")).thenReturn("example.com");
+ Mockito.when(mockReq.getHeader("Access-Control-Request-Method")).thenReturn("GET");
+ Mockito.when(mockReq.getHeader("Access-Control-Request-Headers"))
+ .thenReturn("X-Requested-With");
+
+ // Objects to verify interactions based on request
+ HttpServletResponseForRouterTest mockRes = new HttpServletResponseForRouterTest();
+ FilterChain mockChain = Mockito.mock(FilterChain.class);
+
+ // Object under test
+ filter.doFilter(mockReq, mockRes, mockChain);
+
+ // Why is 5, because when Filter passes,
+ // CrossOriginFilter will set 5 values to Map
+ Assert.assertEquals(5, mockRes.getHeaders().size());
+ String allowResult = mockRes.getHeader("Access-Control-Allow-Credentials");
+ Assert.assertEquals("true", allowResult);
+
+ // 2. Simulate [example.org] for access
+ HttpServletRequest mockReq2 = Mockito.mock(HttpServletRequest.class);
+ Mockito.when(mockReq2.getHeader("Origin")).thenReturn("example.org");
+ Mockito.when(mockReq2.getHeader("Access-Control-Request-Method")).thenReturn("GET");
+ Mockito.when(mockReq2.getHeader("Access-Control-Request-Headers"))
+ .thenReturn("X-Requested-With");
+
+ // Objects to verify interactions based on request
+ HttpServletResponseForRouterTest mockRes2 = new HttpServletResponseForRouterTest();
+ FilterChain mockChain2 = Mockito.mock(FilterChain.class);
+
+ // Object under test
+ filter.doFilter(mockReq2, mockRes2, mockChain2);
+
+ // Why is 0, because when the Filter fails,
+ // CrossOriginFilter will not set any value
+ Assert.assertEquals(0, mockRes2.getHeaders().size());
+
+ router.stop();
+ }
+
+ private class HttpServletResponseForRouterTest extends HttpServletResponseImpl {
+ private final Map<String, String> headers = new HashMap<>(1);
+ @Override
+ public void setHeader(String name, String value) {
+ headers.put(name, value);
+ }
+
+ public String getHeader(String name) {
+ return headers.get(name);
+ }
+
+ public Map<String, String> getHeaders() {
+ return headers;
+ }
+ }
+
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index eae1d8d6fe3..b1791551b2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -284,6 +284,16 @@ Kerberos supported in federation.
| `yarn.router.kerberos.principal` | | The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all Routers in setup. |
| `yarn.router.kerberos.principal.hostname` | | Optional. The hostname for the Router containing this configuration file. Will be different for each machine. Defaults to current hostname. |
+Enabling CORS support:
+
+To enable cross-origin support (CORS) for the Yarn Router, please set the following configuration parameters:
+
+| Property | Example | Description |
+| ----------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| `hadoop.http.filter.initializers` | `org.apache.hadoop.security.HttpCrossOriginFilterInitializer` | Optional. Set the filter to HttpCrossOriginFilterInitializer, Configure this parameter in core-site.xml. |
+| `yarn.router.webapp.cross-origin.enabled` | `true` | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. |
+
+
###ON NMs:
These are extra configurations that should appear in the **conf/yarn-site.xml** at each NodeManager.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
index e05f0256bf9..b4d5386a798 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
@@ -41,6 +41,7 @@ origin (CORS) support.
| `yarn.timeline-service.http-cross-origin.enabled` | true | Enable CORS support for Timeline Server |
| `yarn.resourcemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Resource Manager |
| `yarn.nodemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Node Manager |
+| `yarn.router.webapp.cross-origin.enabled` | true | Enable CORS support for Yarn Router |
Also please ensure that CORS related configurations are enabled in `core-site.xml`.
Kindly refer [here](../../hadoop-project-dist/hadoop-common/HttpAuthentication.html)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org