You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2017/08/22 13:35:04 UTC
[29/51] [abbrv] hadoop git commit: YARN-6638. [ATSv2 Security]
Timeline reader side changes for loading auth filters and principals.
Contributed by Varun Saxena
YARN-6638. [ATSv2 Security] Timeline reader side changes for loading auth filters and principals. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e6324f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e6324f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e6324f7
Branch: refs/heads/YARN-5355
Commit: 6e6324f717a5b0825dd7d53f20eb4b0d72d73b16
Parents: a0121d4
Author: Jian He <ji...@apache.org>
Authored: Fri Jun 9 13:42:38 2017 -0700
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Aug 22 19:03:03 2017 +0530
----------------------------------------------------------------------
...TimelineAuthenticationFilterInitializer.java | 69 ++++++++++---------
.../AbstractTimelineReaderHBaseTestBase.java | 11 +--
.../reader/TimelineReaderServer.java | 70 +++++++++++---------
...neReaderAuthenticationFilterInitializer.java | 53 +++++++++++++++
.../reader/security/package-info.java | 25 +++++++
5 files changed, 160 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e6324f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
index 4e7c29a..06f9868e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
@@ -51,30 +51,18 @@ import java.util.Map;
public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
/**
- * The configuration prefix of timeline HTTP authentication
+ * The configuration prefix of timeline HTTP authentication.
*/
public static final String PREFIX = "yarn.timeline-service.http-authentication.";
@VisibleForTesting
Map<String, String> filterConfig;
- /**
- * Initializes {@link TimelineAuthenticationFilter}
- * <p>
- * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
- * configuration properties prefixed with {@value #PREFIX}
- *
- * @param container
- * The filter container
- * @param conf
- * Configuration for run-time parameters
- */
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
+ protected void setAuthFilterConfig(Configuration conf) {
filterConfig = new HashMap<String, String>();
// setting the cookie path to root '/' so it is used for all resources.
- filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/");
+ filterConfig.put(AuthenticationFilter.COOKIE_PATH, "/");
for (Map.Entry<String, String> entry : conf) {
String name = entry.getKey();
@@ -95,6 +83,41 @@ public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
}
}
+ // Resolve _HOST into bind address
+ String bindAddress = conf.get(HttpServer2.BIND_ADDRESS);
+ String principal =
+ filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
+ if (principal != null) {
+ try {
+ principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
+ } catch (IOException ex) {
+ throw new RuntimeException("Could not resolve Kerberos principal " +
+ "name: " + ex.toString(), ex);
+ }
+ filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL,
+ principal);
+ }
+ }
+
+ protected Map<String, String> getFilterConfig() {
+ return filterConfig;
+ }
+
+ /**
+ * Initializes {@link TimelineAuthenticationFilter}
+ * <p>
+ * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
+ * configuration properties prefixed with {@value #PREFIX}
+ *
+ * @param container
+ * The filter container
+ * @param conf
+ * Configuration for run-time parameters
+ */
+ @Override
+ public void initFilter(FilterContainer container, Configuration conf) {
+ setAuthFilterConfig(conf);
+
String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE);
if (authType.equals(PseudoAuthenticationHandler.TYPE)) {
filterConfig.put(AuthenticationFilter.AUTH_TYPE,
@@ -102,23 +125,7 @@ public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
} else if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
filterConfig.put(AuthenticationFilter.AUTH_TYPE,
KerberosDelegationTokenAuthenticationHandler.class.getName());
-
- // Resolve _HOST into bind address
- String bindAddress = conf.get(HttpServer2.BIND_ADDRESS);
- String principal =
- filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
- if (principal != null) {
- try {
- principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
- } catch (IOException ex) {
- throw new RuntimeException(
- "Could not resolve Kerberos principal name: " + ex.toString(), ex);
- }
- filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL,
- principal);
- }
}
-
filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND,
TimelineDelegationTokenIdentifier.KIND_NAME.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e6324f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java
index 7853c94..3519c3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java
@@ -88,13 +88,14 @@ public abstract class AbstractTimelineReaderHBaseTestBase {
config.setInt("hfile.format.version", 3);
server = new TimelineReaderServer() {
@Override
- protected void setupOptions(Configuration conf) {
- // The parent code tries to use HttpServer2 from this version of
- // Hadoop, but the tests are loading in HttpServer2 from
- // ${hbase-compatible-hadoop.version}. This version uses Jetty 9
+ protected void addFilters(Configuration conf) {
+ // The parent code uses hadoop-common jar from this version of
+ // Hadoop, but the tests are using hadoop-common jar from
+ // ${hbase-compatible-hadoop.version}. This version uses Jetty 9
// while ${hbase-compatible-hadoop.version} uses Jetty 6, and there
// are many differences, including classnames and packages.
- // We do nothing here, so that we don't cause a NoSuchMethodError.
+ // We do nothing here, so that we don't cause a NoSuchMethodError or
+ // NoClassDefFoundError.
// Once ${hbase-compatible-hadoop.version} is changed to Hadoop 3,
// we should be able to remove this @Override.
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e6324f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index 6cdf937..61f2425 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -18,19 +18,18 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
-
+import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.LinkedHashSet;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ReflectionUtils;
@@ -40,7 +39,9 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -71,6 +72,17 @@ public class TimelineReaderServer extends CompositeService {
if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
throw new YarnException("timeline service v.2 is not enabled");
}
+ InetSocketAddress bindAddr = conf.getSocketAddr(
+ YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
+ // Login from keytab if security is enabled.
+ try {
+ SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB,
+ YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, bindAddr.getHostName());
+ } catch(IOException e) {
+ throw new YarnRuntimeException("Failed to login from keytab", e);
+ }
TimelineReader timelineReaderStore = createTimelineReaderStore(conf);
timelineReaderStore.init(conf);
@@ -130,29 +142,39 @@ public class TimelineReaderServer extends CompositeService {
super.serviceStop();
}
- private void startTimelineReaderWebApp() {
- Configuration conf = getConfig();
- String bindAddress = WebAppUtils.getWebAppBindURL(conf,
- YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
- WebAppUtils.getTimelineReaderWebAppURL(conf));
- LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress);
+ protected void addFilters(Configuration conf) {
boolean enableCorsFilter = conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED,
YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT);
- // setup CORS
+ // Setup CORS
if (enableCorsFilter) {
conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+ HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
}
+ String initializers = conf.get("hadoop.http.filter.initializers", "");
+ Set<String> defaultInitializers = new LinkedHashSet<String>();
+ if (!initializers.contains(
+ TimelineReaderAuthenticationFilterInitializer.class.getName())) {
+ defaultInitializers.add(
+ TimelineReaderAuthenticationFilterInitializer.class.getName());
+ }
+ TimelineServerUtils.setTimelineFilters(
+ conf, initializers, defaultInitializers);
+ }
+
+ private void startTimelineReaderWebApp() {
+ Configuration conf = getConfig();
+ addFilters(conf);
+ String bindAddress = WebAppUtils.getWebAppBindURL(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ WebAppUtils.getTimelineReaderWebAppURL(conf));
+ LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress);
try {
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("timeline")
.setConf(conf)
.addEndpoint(URI.create("http://" + bindAddress));
readerWebServer = builder.build();
-
- setupOptions(conf);
-
readerWebServer.addJerseyResourcePackage(
TimelineReaderWebServices.class.getPackage().getName() + ";"
+ GenericExceptionHandler.class.getPackage().getName() + ";"
@@ -168,22 +190,6 @@ public class TimelineReaderServer extends CompositeService {
}
}
- /**
- * Sets up some options and filters.
- *
- * @param conf Configuration
- */
- protected void setupOptions(Configuration conf) {
- Map<String, String> options = new HashMap<>();
- String username = conf.get(HADOOP_HTTP_STATIC_USER,
- DEFAULT_HADOOP_HTTP_STATIC_USER);
- options.put(HADOOP_HTTP_STATIC_USER, username);
- HttpServer2.defineFilter(readerWebServer.getWebAppContext(),
- "static_user_filter_timeline",
- StaticUserWebFilter.StaticUserFilter.class.getName(),
- options, new String[] {"/*"});
- }
-
@VisibleForTesting
public int getWebServerPort() {
return readerWebServer.getConnectorAddress(0).getPort();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e6324f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java
new file mode 100644
index 0000000..e0e1f4d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.FilterContainer;
+import org.apache.hadoop.security.AuthenticationWithProxyUserFilter;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
+
+/**
+ * Filter initializer to initialize {@link AuthenticationWithProxyUserFilter}
+ * for ATSv2 timeline reader server with timeline service specific
+ * configurations.
+ */
+public class TimelineReaderAuthenticationFilterInitializer extends
+ TimelineAuthenticationFilterInitializer{
+
+ /**
+ * Initializes {@link AuthenticationWithProxyUserFilter}
+ * <p>
+ * Propagates to {@link AuthenticationWithProxyUserFilter} configuration all
+ * YARN configuration properties prefixed with
+ * {@value TimelineAuthenticationFilterInitializer#PREFIX}.
+ *
+ * @param container
+ * The filter container
+ * @param conf
+ * Configuration for run-time parameters
+ */
+ @Override
+ public void initFilter(FilterContainer container, Configuration conf) {
+ setAuthFilterConfig(conf);
+ container.addGlobalFilter("Timeline Reader Authentication Filter",
+ AuthenticationWithProxyUserFilter.class.getName(),
+ getFilterConfig());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e6324f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java
new file mode 100644
index 0000000..5888c98
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.server.timelineservice.reader.security contains
+ * classes to be used to support SPNEGO authentication for timeline reader.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.server.timelineservice.reader.security;
+import org.apache.hadoop.classification.InterfaceAudience;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org