You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2017/08/22 13:35:03 UTC
[28/51] [abbrv] hadoop git commit: YARN-5647. [ATSv2 Security]
Collector side changes for loading auth filters and principals. Contributed
by Varun Saxena
YARN-5647. [ATSv2 Security] Collector side changes for loading auth filters and principals. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0121d46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0121d46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0121d46
Branch: refs/heads/YARN-5355
Commit: a0121d46748c914410b84c5fcda9e3148204de64
Parents: 3832fd9
Author: Jian He <ji...@apache.org>
Authored: Wed Jun 7 13:45:34 2017 -0700
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Aug 22 19:03:03 2017 +0530
----------------------------------------------------------------------
.../ApplicationHistoryServer.java | 79 ++---
.../security/TimelineAuthenticationFilter.java | 49 ---
...TimelineAuthenticationFilterInitializer.java | 129 -------
...lineDelegationTokenSecretManagerService.java | 240 --------------
...neV1DelegationTokenSecretManagerService.java | 225 +++++++++++++
.../TestTimelineAuthenticationFilter.java | 323 ------------------
.../TestTimelineAuthenticationFilterForV1.java | 332 +++++++++++++++++++
...TimelineAuthenticationFilterInitializer.java | 76 -----
.../security/TimelineAuthenticationFilter.java | 55 +++
...TimelineAuthenticationFilterInitializer.java | 129 +++++++
...elineDelgationTokenSecretManagerService.java | 83 +++++
.../server/timeline/security/package-info.java | 26 ++
.../util/timeline/TimelineServerUtils.java | 92 +++++
.../yarn/server/util/timeline/package-info.java | 25 ++
...TimelineAuthenticationFilterInitializer.java | 76 +++++
.../collector/NodeTimelineCollectorManager.java | 66 +++-
.../PerNodeTimelineCollectorsAuxService.java | 5 +-
.../collector/TimelineCollectorManager.java | 6 +-
...neV2DelegationTokenSecretManagerService.java | 78 +++++
.../timelineservice/security/package-info.java | 25 ++
20 files changed, 1227 insertions(+), 892 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 85e5f2d..4e3a1e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -20,14 +20,14 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
@@ -47,10 +47,9 @@ import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineV1DelegationTokenSecretManagerService;
import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitializer;
+import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -75,7 +74,7 @@ public class ApplicationHistoryServer extends CompositeService {
private ApplicationACLsManager aclsManager;
private ApplicationHistoryManager historyManager;
private TimelineStore timelineStore;
- private TimelineDelegationTokenSecretManagerService secretManagerService;
+ private TimelineV1DelegationTokenSecretManagerService secretManagerService;
private TimelineDataManager timelineDataManager;
private WebApp webApp;
private JvmPauseMonitor pauseMonitor;
@@ -223,9 +222,9 @@ public class ApplicationHistoryServer extends CompositeService {
TimelineStore.class), conf);
}
- private TimelineDelegationTokenSecretManagerService
+ private TimelineV1DelegationTokenSecretManagerService
createTimelineDelegationTokenSecretManagerService(Configuration conf) {
- return new TimelineDelegationTokenSecretManagerService();
+ return new TimelineV1DelegationTokenSecretManagerService();
}
private TimelineDataManager createTimelineDataManager(Configuration conf) {
@@ -237,63 +236,33 @@ public class ApplicationHistoryServer extends CompositeService {
@SuppressWarnings("unchecked")
private void startWebApp() {
Configuration conf = getConfig();
- TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager(
- secretManagerService.getTimelineDelegationTokenSecretManager());
// Always load pseudo authentication filter to parse "user.name" in an URL
// to identify a HTTP request's user in insecure mode.
// When Kerberos authentication type is set (i.e., secure mode is turned on),
// the customized filter will be loaded by the timeline server to do Kerberos
// + DT authentication.
- String initializers = conf.get("hadoop.http.filter.initializers");
- boolean modifiedInitializers = false;
-
- initializers =
- initializers == null || initializers.length() == 0 ? "" : initializers;
-
+ String initializers = conf.get("hadoop.http.filter.initializers", "");
+ Set<String> defaultInitializers = new LinkedHashSet<String>();
+ // Add CORS filter
if (!initializers.contains(CrossOriginFilterInitializer.class.getName())) {
- if(conf.getBoolean(YarnConfiguration
- .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration
- .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) {
- if (initializers.contains(HttpCrossOriginFilterInitializer.class.getName())) {
- initializers =
- initializers.replaceAll(HttpCrossOriginFilterInitializer.class.getName(),
+ if(conf.getBoolean(YarnConfiguration.
+ TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED,
+ YarnConfiguration.
+ TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) {
+ if (initializers.contains(
+ HttpCrossOriginFilterInitializer.class.getName())) {
+ initializers = initializers.replaceAll(
+ HttpCrossOriginFilterInitializer.class.getName(),
CrossOriginFilterInitializer.class.getName());
+ } else {
+ defaultInitializers.add(CrossOriginFilterInitializer.class.getName());
}
- else {
- if (initializers.length() != 0) {
- initializers += ",";
- }
- initializers += CrossOriginFilterInitializer.class.getName();
- }
- modifiedInitializers = true;
- }
- }
-
- if (!initializers.contains(TimelineAuthenticationFilterInitializer.class
- .getName())) {
- if (initializers.length() != 0) {
- initializers += ",";
- }
- initializers += TimelineAuthenticationFilterInitializer.class.getName();
- modifiedInitializers = true;
- }
-
- String[] parts = initializers.split(",");
- ArrayList<String> target = new ArrayList<String>();
- for (String filterInitializer : parts) {
- filterInitializer = filterInitializer.trim();
- if (filterInitializer.equals(AuthenticationFilterInitializer.class
- .getName())) {
- modifiedInitializers = true;
- continue;
}
- target.add(filterInitializer);
- }
- String actualInitializers =
- org.apache.commons.lang.StringUtils.join(target, ",");
- if (modifiedInitializers) {
- conf.set("hadoop.http.filter.initializers", actualInitializers);
}
+ TimelineServerUtils.addTimelineAuthFilter(
+ initializers, defaultInitializers, secretManagerService);
+ TimelineServerUtils.setTimelineFilters(
+ conf, initializers, defaultInitializers);
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
deleted file mode 100644
index ad8dc2c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timeline.security;
-
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService.TimelineDelegationTokenSecretManager;
-
-@Private
-@Unstable
-public class TimelineAuthenticationFilter
- extends DelegationTokenAuthenticationFilter {
-
- private static TimelineDelegationTokenSecretManager secretManager;
-
- @Override
- public void init(FilterConfig filterConfig) throws ServletException {
- filterConfig.getServletContext().setAttribute(
- DelegationTokenAuthenticationFilter.DELEGATION_TOKEN_SECRET_MANAGER_ATTR,
- secretManager);
- super.init(filterConfig);
- }
-
- public static void setTimelineDelegationTokenSecretManager(
- TimelineDelegationTokenSecretManager secretManager) {
- TimelineAuthenticationFilter.secretManager = secretManager;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
deleted file mode 100644
index 4e7c29a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timeline.security;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.FilterContainer;
-import org.apache.hadoop.http.FilterInitializer;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
-import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
-import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
-import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
-import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Initializes {@link TimelineAuthenticationFilter} which provides support for
- * Kerberos HTTP SPNEGO authentication.
- * <p>
- * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the
- * timeline server.
- * <p>
- * Refer to the {@code core-default.xml} file, after the comment 'HTTP
- * Authentication' for details on the configuration options. All related
- * configuration properties have {@code hadoop.http.authentication.} as prefix.
- */
-public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
-
- /**
- * The configuration prefix of timeline HTTP authentication
- */
- public static final String PREFIX = "yarn.timeline-service.http-authentication.";
-
- @VisibleForTesting
- Map<String, String> filterConfig;
-
- /**
- * Initializes {@link TimelineAuthenticationFilter}
- * <p>
- * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
- * configuration properties prefixed with {@value #PREFIX}
- *
- * @param container
- * The filter container
- * @param conf
- * Configuration for run-time parameters
- */
- @Override
- public void initFilter(FilterContainer container, Configuration conf) {
- filterConfig = new HashMap<String, String>();
-
- // setting the cookie path to root '/' so it is used for all resources.
- filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/");
-
- for (Map.Entry<String, String> entry : conf) {
- String name = entry.getKey();
- if (name.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) {
- String value = conf.get(name);
- name = name.substring("hadoop.".length());
- filterConfig.put(name, value);
- }
- }
- for (Map.Entry<String, String> entry : conf) {
- String name = entry.getKey();
- if (name.startsWith(PREFIX)) {
- // yarn.timeline-service.http-authentication.proxyuser will override
- // hadoop.proxyuser
- String value = conf.get(name);
- name = name.substring(PREFIX.length());
- filterConfig.put(name, value);
- }
- }
-
- String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE);
- if (authType.equals(PseudoAuthenticationHandler.TYPE)) {
- filterConfig.put(AuthenticationFilter.AUTH_TYPE,
- PseudoDelegationTokenAuthenticationHandler.class.getName());
- } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
- filterConfig.put(AuthenticationFilter.AUTH_TYPE,
- KerberosDelegationTokenAuthenticationHandler.class.getName());
-
- // Resolve _HOST into bind address
- String bindAddress = conf.get(HttpServer2.BIND_ADDRESS);
- String principal =
- filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
- if (principal != null) {
- try {
- principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
- } catch (IOException ex) {
- throw new RuntimeException(
- "Could not resolve Kerberos principal name: " + ex.toString(), ex);
- }
- filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL,
- principal);
- }
- }
-
- filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND,
- TimelineDelegationTokenIdentifier.KIND_NAME.toString());
-
- container.addGlobalFilter("Timeline Authentication Filter",
- TimelineAuthenticationFilter.class.getName(),
- filterConfig);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java
deleted file mode 100644
index 0c6892a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timeline.security;
-
-import java.io.IOException;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
-import org.apache.hadoop.security.token.delegation.DelegationKey;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore;
-import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
-import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The service wrapper of {@link TimelineDelegationTokenSecretManager}
- */
-@Private
-@Unstable
-public class TimelineDelegationTokenSecretManagerService extends
- AbstractService {
-
- private TimelineDelegationTokenSecretManager secretManager = null;
- private TimelineStateStore stateStore = null;
-
- public TimelineDelegationTokenSecretManagerService() {
- super(TimelineDelegationTokenSecretManagerService.class.getName());
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED)) {
- stateStore = createStateStore(conf);
- stateStore.init(conf);
- }
-
- long secretKeyInterval =
- conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL,
- YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL);
- long tokenMaxLifetime =
- conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME,
- YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME);
- long tokenRenewInterval =
- conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL,
- YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL);
- secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval,
- tokenMaxLifetime, tokenRenewInterval, 3600000, stateStore);
- super.init(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- if (stateStore != null) {
- stateStore.start();
- TimelineServiceState state = stateStore.loadState();
- secretManager.recover(state);
- }
-
- secretManager.startThreads();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (stateStore != null) {
- stateStore.stop();
- }
-
- secretManager.stopThreads();
- super.stop();
- }
-
- protected TimelineStateStore createStateStore(
- Configuration conf) {
- return ReflectionUtils.newInstance(
- conf.getClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
- LeveldbTimelineStateStore.class,
- TimelineStateStore.class), conf);
- }
-
- /**
- * Ge the instance of {link #TimelineDelegationTokenSecretManager}
- *
- * @return the instance of {link #TimelineDelegationTokenSecretManager}
- */
- public TimelineDelegationTokenSecretManager
- getTimelineDelegationTokenSecretManager() {
- return secretManager;
- }
-
- @Private
- @Unstable
- public static class TimelineDelegationTokenSecretManager extends
- AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(TimelineDelegationTokenSecretManager.class);
-
- private TimelineStateStore stateStore;
-
- /**
- * Create a timeline secret manager
- * @param delegationKeyUpdateInterval the number of milliseconds for rolling
- * new secret keys.
- * @param delegationTokenMaxLifetime the maximum lifetime of the delegation
- * tokens in milliseconds
- * @param delegationTokenRenewInterval how often the tokens must be renewed
- * in milliseconds
- * @param delegationTokenRemoverScanInterval how often the tokens are
- * scanned for expired tokens in milliseconds
- * @param stateStore timeline service state store
- */
- public TimelineDelegationTokenSecretManager(
- long delegationKeyUpdateInterval,
- long delegationTokenMaxLifetime,
- long delegationTokenRenewInterval,
- long delegationTokenRemoverScanInterval,
- TimelineStateStore stateStore) {
- super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
- delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
- this.stateStore = stateStore;
- }
-
- @Override
- public TimelineDelegationTokenIdentifier createIdentifier() {
- return new TimelineDelegationTokenIdentifier();
- }
-
- @Override
- protected void storeNewMasterKey(DelegationKey key) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing master key " + key.getKeyId());
- }
- try {
- if (stateStore != null) {
- stateStore.storeTokenMasterKey(key);
- }
- } catch (IOException e) {
- LOG.error("Unable to store master key " + key.getKeyId(), e);
- }
- }
-
- @Override
- protected void removeStoredMasterKey(DelegationKey key) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removing master key " + key.getKeyId());
- }
- try {
- if (stateStore != null) {
- stateStore.removeTokenMasterKey(key);
- }
- } catch (IOException e) {
- LOG.error("Unable to remove master key " + key.getKeyId(), e);
- }
- }
-
- @Override
- protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId,
- long renewDate) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing token " + tokenId.getSequenceNumber());
- }
- try {
- if (stateStore != null) {
- stateStore.storeToken(tokenId, renewDate);
- }
- } catch (IOException e) {
- LOG.error("Unable to store token " + tokenId.getSequenceNumber(), e);
- }
- }
-
- @Override
- protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId)
- throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing token " + tokenId.getSequenceNumber());
- }
- try {
- if (stateStore != null) {
- stateStore.removeToken(tokenId);
- }
- } catch (IOException e) {
- LOG.error("Unable to remove token " + tokenId.getSequenceNumber(), e);
- }
- }
-
- @Override
- protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId,
- long renewDate) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Updating token " + tokenId.getSequenceNumber());
- }
- try {
- if (stateStore != null) {
- stateStore.updateToken(tokenId, renewDate);
- }
- } catch (IOException e) {
- LOG.error("Unable to update token " + tokenId.getSequenceNumber(), e);
- }
- }
-
- public void recover(TimelineServiceState state) throws IOException {
- LOG.info("Recovering " + getClass().getSimpleName());
- for (DelegationKey key : state.getTokenMasterKeyState()) {
- addKey(key);
- }
- this.delegationTokenSequenceNumber = state.getLatestSequenceNumber();
- for (Entry<TimelineDelegationTokenIdentifier, Long> entry :
- state.getTokenState().entrySet()) {
- addPersistedDelegationToken(entry.getKey(), entry.getValue());
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java
new file mode 100644
index 0000000..85d8cca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.security;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The service wrapper of {@link TimelineV1DelegationTokenSecretManager}.
+ */
+@Private
+@Unstable
+public class TimelineV1DelegationTokenSecretManagerService extends
+ TimelineDelgationTokenSecretManagerService {
+ private TimelineStateStore stateStore = null;
+
+ public TimelineV1DelegationTokenSecretManagerService() {
+ super(TimelineV1DelegationTokenSecretManagerService.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED)) {
+ stateStore = createStateStore(conf);
+ stateStore.init(conf);
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ if (stateStore != null) {
+ stateStore.start();
+ TimelineServiceState state = stateStore.loadState();
+ ((TimelineV1DelegationTokenSecretManager)
+ getTimelineDelegationTokenSecretManager()).recover(state);
+ }
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (stateStore != null) {
+ stateStore.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ protected AbstractDelegationTokenSecretManager
+ <TimelineDelegationTokenIdentifier>
+ createTimelineDelegationTokenSecretManager(long secretKeyInterval,
+ long tokenMaxLifetime, long tokenRenewInterval,
+ long tokenRemovalScanInterval) {
+ return new TimelineV1DelegationTokenSecretManager(secretKeyInterval,
+ tokenMaxLifetime, tokenRenewInterval, tokenRemovalScanInterval,
+ stateStore);
+ }
+
+ protected TimelineStateStore createStateStore(
+ Configuration conf) {
+ return ReflectionUtils.newInstance(
+ conf.getClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+ LeveldbTimelineStateStore.class,
+ TimelineStateStore.class), conf);
+ }
+
+ /**
+ * Delegation token secret manager for ATSv1 and ATSv1.5.
+ */
+ @Private
+ @Unstable
+ public static class TimelineV1DelegationTokenSecretManager extends
+ AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TimelineV1DelegationTokenSecretManager.class);
+
+ private TimelineStateStore stateStore;
+
+ /**
+ * Create a timeline v1 secret manager.
+ * @param delegationKeyUpdateInterval the number of milliseconds for rolling
+ * new secret keys.
+ * @param delegationTokenMaxLifetime the maximum lifetime of the delegation
+ * tokens in milliseconds
+ * @param delegationTokenRenewInterval how often the tokens must be renewed
+ * in milliseconds
+ * @param delegationTokenRemoverScanInterval how often the tokens are
+ * scanned for expired tokens in milliseconds
+ * @param stateStore timeline service state store
+ */
+ public TimelineV1DelegationTokenSecretManager(
+ long delegationKeyUpdateInterval,
+ long delegationTokenMaxLifetime,
+ long delegationTokenRenewInterval,
+ long delegationTokenRemoverScanInterval,
+ TimelineStateStore stateStore) {
+ super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
+ delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+ this.stateStore = stateStore;
+ }
+
+ @Override
+ public TimelineDelegationTokenIdentifier createIdentifier() {
+ return new TimelineDelegationTokenIdentifier();
+ }
+
+ @Override
+ protected void storeNewMasterKey(DelegationKey key) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing master key " + key.getKeyId());
+ }
+ try {
+ if (stateStore != null) {
+ stateStore.storeTokenMasterKey(key);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to store master key " + key.getKeyId(), e);
+ }
+ }
+
+ @Override
+ protected void removeStoredMasterKey(DelegationKey key) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing master key " + key.getKeyId());
+ }
+ try {
+ if (stateStore != null) {
+ stateStore.removeTokenMasterKey(key);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to remove master key " + key.getKeyId(), e);
+ }
+ }
+
+ @Override
+ protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId,
+ long renewDate) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing token " + tokenId.getSequenceNumber());
+ }
+ try {
+ if (stateStore != null) {
+ stateStore.storeToken(tokenId, renewDate);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to store token " + tokenId.getSequenceNumber(), e);
+ }
+ }
+
+ @Override
+ protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing token " + tokenId.getSequenceNumber());
+ }
+ try {
+ if (stateStore != null) {
+ stateStore.removeToken(tokenId);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to remove token " + tokenId.getSequenceNumber(), e);
+ }
+ }
+
+ @Override
+ protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId,
+ long renewDate) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating token " + tokenId.getSequenceNumber());
+ }
+ try {
+ if (stateStore != null) {
+ stateStore.updateToken(tokenId, renewDate);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to update token " + tokenId.getSequenceNumber(), e);
+ }
+ }
+
+ public void recover(TimelineServiceState state) throws IOException {
+ LOG.info("Recovering " + getClass().getSimpleName());
+ for (DelegationKey key : state.getTokenMasterKeyState()) {
+ addKey(key);
+ }
+ this.delegationTokenSequenceNumber = state.getLatestSequenceNumber();
+ for (Entry<TimelineDelegationTokenIdentifier, Long> entry :
+ state.getTokenState().entrySet()) {
+ addPersistedDelegationToken(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
deleted file mode 100644
index 063f512..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timeline.security;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.minikdc.MiniKdc;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.KerberosTestUtils;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
-import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
-import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
-import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-@RunWith(Parameterized.class)
-public class TestTimelineAuthenticationFilter {
-
- private static final String FOO_USER = "foo";
- private static final String BAR_USER = "bar";
- private static final String HTTP_USER = "HTTP";
-
- private static final File testRootDir = new File(
- System.getProperty("test.build.dir", "target/test-dir"),
- TestTimelineAuthenticationFilter.class.getName() + "-root");
- private static File httpSpnegoKeytabFile = new File(
- KerberosTestUtils.getKeytabFile());
- private static String httpSpnegoPrincipal =
- KerberosTestUtils.getServerPrincipal();
- private static final String BASEDIR =
- System.getProperty("test.build.dir", "target/test-dir") + "/"
- + TestTimelineAuthenticationFilter.class.getSimpleName();
-
- @Parameterized.Parameters
- public static Collection<Object[]> withSsl() {
- return Arrays.asList(new Object[][] { { false }, { true } });
- }
-
- private static MiniKdc testMiniKDC;
- private static String keystoresDir;
- private static String sslConfDir;
- private static ApplicationHistoryServer testTimelineServer;
- private static Configuration conf;
- private static boolean withSsl;
-
- public TestTimelineAuthenticationFilter(boolean withSsl) {
- TestTimelineAuthenticationFilter.withSsl = withSsl;
- }
-
- @BeforeClass
- public static void setup() {
- try {
- testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
- testMiniKDC.start();
- testMiniKDC.createPrincipal(
- httpSpnegoKeytabFile, HTTP_USER + "/localhost");
- } catch (Exception e) {
- assertTrue("Couldn't setup MiniKDC", false);
- }
-
- try {
- testTimelineServer = new ApplicationHistoryServer();
- conf = new Configuration(false);
- conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type",
- "kerberos");
- conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
- KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
- conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
- KerberosAuthenticationHandler.KEYTAB,
- httpSpnegoKeytabFile.getAbsolutePath());
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "kerberos");
- conf.set(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL,
- httpSpnegoPrincipal);
- conf.set(YarnConfiguration.TIMELINE_SERVICE_KEYTAB,
- httpSpnegoKeytabFile.getAbsolutePath());
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
- MemoryTimelineStore.class, TimelineStore.class);
- conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
- "localhost:10200");
- conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
- "localhost:8188");
- conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
- "localhost:8190");
- conf.set("hadoop.proxyuser.HTTP.hosts", "*");
- conf.set("hadoop.proxyuser.HTTP.users", FOO_USER);
- conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 1);
-
- if (withSsl) {
- conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
- HttpConfig.Policy.HTTPS_ONLY.name());
- File base = new File(BASEDIR);
- FileUtil.fullyDelete(base);
- base.mkdirs();
- keystoresDir = new File(BASEDIR).getAbsolutePath();
- sslConfDir =
- KeyStoreTestUtil.getClasspathDir(TestTimelineAuthenticationFilter.class);
- KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
- }
-
- UserGroupInformation.setConfiguration(conf);
- testTimelineServer.init(conf);
- testTimelineServer.start();
- } catch (Exception e) {
- assertTrue("Couldn't setup TimelineServer", false);
- }
- }
-
- private TimelineClient createTimelineClientForUGI() {
- TimelineClient client = TimelineClient.createTimelineClient();
- client.init(conf);
- client.start();
- return client;
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- if (testMiniKDC != null) {
- testMiniKDC.stop();
- }
-
- if (testTimelineServer != null) {
- testTimelineServer.stop();
- }
-
- if (withSsl) {
- KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
- File base = new File(BASEDIR);
- FileUtil.fullyDelete(base);
- }
- }
-
- @Test
- public void testPutTimelineEntities() throws Exception {
- KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- TimelineClient client = createTimelineClientForUGI();
- TimelineEntity entityToStore = new TimelineEntity();
- entityToStore.setEntityType(
- TestTimelineAuthenticationFilter.class.getName());
- entityToStore.setEntityId("entity1");
- entityToStore.setStartTime(0L);
- TimelinePutResponse putResponse = client.putEntities(entityToStore);
- Assert.assertEquals(0, putResponse.getErrors().size());
- TimelineEntity entityToRead =
- testTimelineServer.getTimelineStore().getEntity(
- "entity1", TestTimelineAuthenticationFilter.class.getName(), null);
- Assert.assertNotNull(entityToRead);
- return null;
- }
- });
- }
-
- @Test
- public void testPutDomains() throws Exception {
- KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- TimelineClient client = createTimelineClientForUGI();
- TimelineDomain domainToStore = new TimelineDomain();
- domainToStore.setId(TestTimelineAuthenticationFilter.class.getName());
- domainToStore.setReaders("*");
- domainToStore.setWriters("*");
- client.putDomain(domainToStore);
- TimelineDomain domainToRead =
- testTimelineServer.getTimelineStore().getDomain(
- TestTimelineAuthenticationFilter.class.getName());
- Assert.assertNotNull(domainToRead);
- return null;
- }
- });
- }
-
- @Test
- public void testDelegationTokenOperations() throws Exception {
- TimelineClient httpUserClient =
- KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<TimelineClient>() {
- @Override
- public TimelineClient call() throws Exception {
- return createTimelineClientForUGI();
- }
- });
- UserGroupInformation httpUser =
- KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<UserGroupInformation>() {
- @Override
- public UserGroupInformation call() throws Exception {
- return UserGroupInformation.getCurrentUser();
- }
- });
- // Let HTTP user to get the delegation for itself
- Token<TimelineDelegationTokenIdentifier> token =
- httpUserClient.getDelegationToken(httpUser.getShortUserName());
- Assert.assertNotNull(token);
- TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
- Assert.assertNotNull(tDT);
- Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
-
- // Renew token
- Assert.assertFalse(token.getService().toString().isEmpty());
- // Renew the token from the token service address
- long renewTime1 = httpUserClient.renewDelegationToken(token);
- Thread.sleep(100);
- token.setService(new Text());
- Assert.assertTrue(token.getService().toString().isEmpty());
- // If the token service address is not avaiable, it still can be renewed
- // from the configured address
- long renewTime2 = httpUserClient.renewDelegationToken(token);
- Assert.assertTrue(renewTime1 < renewTime2);
-
- // Cancel token
- Assert.assertTrue(token.getService().toString().isEmpty());
- // If the token service address is not avaiable, it still can be canceled
- // from the configured address
- httpUserClient.cancelDelegationToken(token);
- // Renew should not be successful because the token is canceled
- try {
- httpUserClient.renewDelegationToken(token);
- Assert.fail();
- } catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains(
- "Renewal request for unknown token"));
- }
-
- // Let HTTP user to get the delegation token for FOO user
- UserGroupInformation fooUgi = UserGroupInformation.createProxyUser(
- FOO_USER, httpUser);
- TimelineClient fooUserClient = fooUgi.doAs(
- new PrivilegedExceptionAction<TimelineClient>() {
- @Override
- public TimelineClient run() throws Exception {
- return createTimelineClientForUGI();
- }
- });
- token = fooUserClient.getDelegationToken(httpUser.getShortUserName());
- Assert.assertNotNull(token);
- tDT = token.decodeIdentifier();
- Assert.assertNotNull(tDT);
- Assert.assertEquals(new Text(FOO_USER), tDT.getOwner());
- Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser());
-
- // Renew token as the renewer
- final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
- renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew);
- renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew);
- Assert.assertTrue(renewTime1 < renewTime2);
-
- // Cancel token
- Assert.assertFalse(tokenToRenew.getService().toString().isEmpty());
- // Cancel the token from the token service address
- fooUserClient.cancelDelegationToken(tokenToRenew);
-
- // Renew should not be successful because the token is canceled
- try {
- httpUserClient.renewDelegationToken(tokenToRenew);
- Assert.fail();
- } catch (Exception e) {
- Assert.assertTrue(
- e.getMessage().contains("Renewal request for unknown token"));
- }
-
- // Let HTTP user to get the delegation token for BAR user
- UserGroupInformation barUgi = UserGroupInformation.createProxyUser(
- BAR_USER, httpUser);
- TimelineClient barUserClient = barUgi.doAs(
- new PrivilegedExceptionAction<TimelineClient>() {
- @Override
- public TimelineClient run() {
- return createTimelineClientForUGI();
- }
- });
-
- try {
- barUserClient.getDelegationToken(httpUser.getShortUserName());
- Assert.fail();
- } catch (Exception e) {
- Assert.assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
new file mode 100644
index 0000000..d918e8d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
@@ -0,0 +1,332 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.security;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.KerberosTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
+import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
+import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Test cases for authentication via TimelineAuthenticationFilter while
+ * publishing entities for ATSv1.
+ */
+@RunWith(Parameterized.class)
+public class TestTimelineAuthenticationFilterForV1 {
+
+ private static final String FOO_USER = "foo";
+ private static final String BAR_USER = "bar";
+ private static final String HTTP_USER = "HTTP";
+
+ private static final File TEST_ROOT_DIR = new File(
+ System.getProperty("test.build.dir", "target/test-dir"),
+ TestTimelineAuthenticationFilterForV1.class.getName() + "-root");
+ private static File httpSpnegoKeytabFile = new File(
+ KerberosTestUtils.getKeytabFile());
+ private static String httpSpnegoPrincipal =
+ KerberosTestUtils.getServerPrincipal();
+ private static final String BASEDIR =
+ System.getProperty("test.build.dir", "target/test-dir") + "/"
+ + TestTimelineAuthenticationFilterForV1.class.getSimpleName();
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> withSsl() {
+ return Arrays.asList(new Object[][] {{false}, {true}});
+ }
+
+ private static MiniKdc testMiniKDC;
+ private static String keystoresDir;
+ private static String sslConfDir;
+ private static ApplicationHistoryServer testTimelineServer;
+ private static Configuration conf;
+ private static boolean withSsl;
+
+ public TestTimelineAuthenticationFilterForV1(boolean withSsl) {
+ TestTimelineAuthenticationFilterForV1.withSsl = withSsl;
+ }
+
+ @BeforeClass
+ public static void setup() {
+ try {
+ testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
+ testMiniKDC.start();
+ testMiniKDC.createPrincipal(
+ httpSpnegoKeytabFile, HTTP_USER + "/localhost");
+ } catch (Exception e) {
+ assertTrue("Couldn't setup MiniKDC", false);
+ }
+
+ try {
+ testTimelineServer = new ApplicationHistoryServer();
+ conf = new Configuration(false);
+ conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type",
+ "kerberos");
+ conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
+ KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
+ conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
+ KerberosAuthenticationHandler.KEYTAB,
+ httpSpnegoKeytabFile.getAbsolutePath());
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL,
+ httpSpnegoPrincipal);
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_KEYTAB,
+ httpSpnegoKeytabFile.getAbsolutePath());
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
+ MemoryTimelineStore.class, TimelineStore.class);
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ "localhost:10200");
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:8188");
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+ "localhost:8190");
+ conf.set("hadoop.proxyuser.HTTP.hosts", "*");
+ conf.set("hadoop.proxyuser.HTTP.users", FOO_USER);
+ conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 1);
+
+ if (withSsl) {
+ conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
+ HttpConfig.Policy.HTTPS_ONLY.name());
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ keystoresDir = new File(BASEDIR).getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(
+ TestTimelineAuthenticationFilterForV1.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+ }
+
+ UserGroupInformation.setConfiguration(conf);
+ testTimelineServer.init(conf);
+ testTimelineServer.start();
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertTrue("Couldn't setup TimelineServer", false);
+ }
+ }
+
+ private TimelineClient createTimelineClientForUGI() {
+ TimelineClient client = TimelineClient.createTimelineClient();
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (testMiniKDC != null) {
+ testMiniKDC.stop();
+ }
+
+ if (testTimelineServer != null) {
+ testTimelineServer.stop();
+ }
+
+ if (withSsl) {
+ KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ }
+ }
+
+ @Test
+ public void testPutTimelineEntities() throws Exception {
+ KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ TimelineClient client = createTimelineClientForUGI();
+ TimelineEntity entityToStore = new TimelineEntity();
+ entityToStore.setEntityType(
+ TestTimelineAuthenticationFilterForV1.class.getName());
+ entityToStore.setEntityId("entity1");
+ entityToStore.setStartTime(0L);
+ TimelinePutResponse putResponse = client.putEntities(entityToStore);
+ Assert.assertEquals(0, putResponse.getErrors().size());
+ TimelineEntity entityToRead =
+ testTimelineServer.getTimelineStore().getEntity("entity1",
+ TestTimelineAuthenticationFilterForV1.class.getName(), null);
+ Assert.assertNotNull(entityToRead);
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testPutDomains() throws Exception {
+ KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ TimelineClient client = createTimelineClientForUGI();
+ TimelineDomain domainToStore = new TimelineDomain();
+ domainToStore.setId(
+ TestTimelineAuthenticationFilterForV1.class.getName());
+ domainToStore.setReaders("*");
+ domainToStore.setWriters("*");
+ client.putDomain(domainToStore);
+ TimelineDomain domainToRead =
+ testTimelineServer.getTimelineStore().getDomain(
+ TestTimelineAuthenticationFilterForV1.class.getName());
+ Assert.assertNotNull(domainToRead);
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testDelegationTokenOperations() throws Exception {
+ TimelineClient httpUserClient =
+ KerberosTestUtils.doAs(HTTP_USER + "/localhost",
+ new Callable<TimelineClient>() {
+ @Override
+ public TimelineClient call() throws Exception {
+ return createTimelineClientForUGI();
+ }
+ });
+ UserGroupInformation httpUser =
+ KerberosTestUtils.doAs(HTTP_USER + "/localhost",
+ new Callable<UserGroupInformation>() {
+ @Override
+ public UserGroupInformation call() throws Exception {
+ return UserGroupInformation.getCurrentUser();
+ }
+ });
+ // Let HTTP user to get the delegation for itself
+ Token<TimelineDelegationTokenIdentifier> token =
+ httpUserClient.getDelegationToken(httpUser.getShortUserName());
+ Assert.assertNotNull(token);
+ TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
+ Assert.assertNotNull(tDT);
+ Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
+
+ // Renew token
+ Assert.assertFalse(token.getService().toString().isEmpty());
+ // Renew the token from the token service address
+ long renewTime1 = httpUserClient.renewDelegationToken(token);
+ Thread.sleep(100);
+ token.setService(new Text());
+ Assert.assertTrue(token.getService().toString().isEmpty());
+ // If the token service address is not avaiable, it still can be renewed
+ // from the configured address
+ long renewTime2 = httpUserClient.renewDelegationToken(token);
+ Assert.assertTrue(renewTime1 < renewTime2);
+
+ // Cancel token
+ Assert.assertTrue(token.getService().toString().isEmpty());
+ // If the token service address is not avaiable, it still can be canceled
+ // from the configured address
+ httpUserClient.cancelDelegationToken(token);
+ // Renew should not be successful because the token is canceled
+ try {
+ httpUserClient.renewDelegationToken(token);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Renewal request for unknown token"));
+ }
+
+ // Let HTTP user to get the delegation token for FOO user
+ UserGroupInformation fooUgi = UserGroupInformation.createProxyUser(
+ FOO_USER, httpUser);
+ TimelineClient fooUserClient = fooUgi.doAs(
+ new PrivilegedExceptionAction<TimelineClient>() {
+ @Override
+ public TimelineClient run() throws Exception {
+ return createTimelineClientForUGI();
+ }
+ });
+ token = fooUserClient.getDelegationToken(httpUser.getShortUserName());
+ Assert.assertNotNull(token);
+ tDT = token.decodeIdentifier();
+ Assert.assertNotNull(tDT);
+ Assert.assertEquals(new Text(FOO_USER), tDT.getOwner());
+ Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser());
+
+ // Renew token as the renewer
+ final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
+ renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew);
+ renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew);
+ Assert.assertTrue(renewTime1 < renewTime2);
+
+ // Cancel token
+ Assert.assertFalse(tokenToRenew.getService().toString().isEmpty());
+ // Cancel the token from the token service address
+ fooUserClient.cancelDelegationToken(tokenToRenew);
+
+ // Renew should not be successful because the token is canceled
+ try {
+ httpUserClient.renewDelegationToken(tokenToRenew);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(
+ e.getMessage().contains("Renewal request for unknown token"));
+ }
+
+ // Let HTTP user to get the delegation token for BAR user
+ UserGroupInformation barUgi = UserGroupInformation.createProxyUser(
+ BAR_USER, httpUser);
+ TimelineClient barUserClient = barUgi.doAs(
+ new PrivilegedExceptionAction<TimelineClient>() {
+ @Override
+ public TimelineClient run() {
+ return createTimelineClientForUGI();
+ }
+ });
+
+ try {
+ barUserClient.getDelegationToken(httpUser.getShortUserName());
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof AuthorizationException ||
+ e.getCause() instanceof AuthenticationException);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java
deleted file mode 100644
index 430911e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timeline.security;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.FilterContainer;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer.PREFIX;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-
-public class TestTimelineAuthenticationFilterInitializer {
-
- @Test
- public void testProxyUserConfiguration() {
- FilterContainer container = Mockito.mock(FilterContainer.class);
- for (int i = 0; i < 3; ++i) {
- Configuration conf = new YarnConfiguration();
- switch (i) {
- case 0:
- // hadoop.proxyuser prefix
- conf.set("hadoop.proxyuser.foo.hosts", "*");
- conf.set("hadoop.proxyuser.foo.users", "*");
- conf.set("hadoop.proxyuser.foo.groups", "*");
- break;
- case 1:
- // yarn.timeline-service.http-authentication.proxyuser prefix
- conf.set(PREFIX + "proxyuser.foo.hosts", "*");
- conf.set(PREFIX + "proxyuser.foo.users", "*");
- conf.set(PREFIX + "proxyuser.foo.groups", "*");
- break;
- case 2:
- // hadoop.proxyuser prefix has been overwritten by
- // yarn.timeline-service.http-authentication.proxyuser prefix
- conf.set("hadoop.proxyuser.foo.hosts", "bar");
- conf.set("hadoop.proxyuser.foo.users", "bar");
- conf.set("hadoop.proxyuser.foo.groups", "bar");
- conf.set(PREFIX + "proxyuser.foo.hosts", "*");
- conf.set(PREFIX + "proxyuser.foo.users", "*");
- conf.set(PREFIX + "proxyuser.foo.groups", "*");
- break;
- default:
- break;
- }
-
- TimelineAuthenticationFilterInitializer initializer =
- new TimelineAuthenticationFilterInitializer();
- initializer.initFilter(container, conf);
- Assert.assertEquals(
- "*", initializer.filterConfig.get("proxyuser.foo.hosts"));
- Assert.assertEquals(
- "*", initializer.filterConfig.get("proxyuser.foo.users"));
- Assert.assertEquals(
- "*", initializer.filterConfig.get("proxyuser.foo.groups"));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
new file mode 100644
index 0000000..f6d1863
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.security;
+
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+
+/**
+ * Timeline authentication filter provides delegation token support for ATSv1
+ * and ATSv2.
+ */
+@Private
+@Unstable
+public class TimelineAuthenticationFilter
+ extends DelegationTokenAuthenticationFilter {
+
+ private static AbstractDelegationTokenSecretManager
+ <TimelineDelegationTokenIdentifier> secretManager;
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ filterConfig.getServletContext().setAttribute(
+ DelegationTokenAuthenticationFilter.
+ DELEGATION_TOKEN_SECRET_MANAGER_ATTR, secretManager);
+ super.init(filterConfig);
+ }
+
+ public static void setTimelineDelegationTokenSecretManager(
+ AbstractDelegationTokenSecretManager
+ <TimelineDelegationTokenIdentifier> secretMgr) {
+ TimelineAuthenticationFilter.secretManager = secretMgr;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
new file mode 100644
index 0000000..4e7c29a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.security;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.FilterContainer;
+import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
+import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Initializes {@link TimelineAuthenticationFilter} which provides support for
+ * Kerberos HTTP SPNEGO authentication.
+ * <p>
+ * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the
+ * timeline server.
+ * <p>
+ * Refer to the {@code core-default.xml} file, after the comment 'HTTP
+ * Authentication' for details on the configuration options. All related
+ * configuration properties have {@code hadoop.http.authentication.} as prefix.
+ */
+public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
+
+ /**
+ * The configuration prefix of timeline HTTP authentication
+ */
+ public static final String PREFIX = "yarn.timeline-service.http-authentication.";
+
+ @VisibleForTesting
+ Map<String, String> filterConfig;
+
+ /**
+ * Initializes {@link TimelineAuthenticationFilter}
+ * <p>
+ * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
+ * configuration properties prefixed with {@value #PREFIX}
+ *
+ * @param container
+ * The filter container
+ * @param conf
+ * Configuration for run-time parameters
+ */
+ @Override
+ public void initFilter(FilterContainer container, Configuration conf) {
+ filterConfig = new HashMap<String, String>();
+
+ // setting the cookie path to root '/' so it is used for all resources.
+ filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/");
+
+ for (Map.Entry<String, String> entry : conf) {
+ String name = entry.getKey();
+ if (name.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) {
+ String value = conf.get(name);
+ name = name.substring("hadoop.".length());
+ filterConfig.put(name, value);
+ }
+ }
+ for (Map.Entry<String, String> entry : conf) {
+ String name = entry.getKey();
+ if (name.startsWith(PREFIX)) {
+ // yarn.timeline-service.http-authentication.proxyuser will override
+ // hadoop.proxyuser
+ String value = conf.get(name);
+ name = name.substring(PREFIX.length());
+ filterConfig.put(name, value);
+ }
+ }
+
+ String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE);
+ if (authType.equals(PseudoAuthenticationHandler.TYPE)) {
+ filterConfig.put(AuthenticationFilter.AUTH_TYPE,
+ PseudoDelegationTokenAuthenticationHandler.class.getName());
+ } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
+ filterConfig.put(AuthenticationFilter.AUTH_TYPE,
+ KerberosDelegationTokenAuthenticationHandler.class.getName());
+
+ // Resolve _HOST into bind address
+ String bindAddress = conf.get(HttpServer2.BIND_ADDRESS);
+ String principal =
+ filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL);
+ if (principal != null) {
+ try {
+ principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
+ } catch (IOException ex) {
+ throw new RuntimeException(
+ "Could not resolve Kerberos principal name: " + ex.toString(), ex);
+ }
+ filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL,
+ principal);
+ }
+ }
+
+ filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND,
+ TimelineDelegationTokenIdentifier.KIND_NAME.toString());
+
+ container.addGlobalFilter("Timeline Authentication Filter",
+ TimelineAuthenticationFilter.class.getName(),
+ filterConfig);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java
new file mode 100644
index 0000000..2e95af2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+
+/**
+ * Abstract implementation of delegation token manager service for different
+ * versions of timeline service.
+ */
+public abstract class TimelineDelgationTokenSecretManagerService extends
+ AbstractService {
+
+ public TimelineDelgationTokenSecretManagerService(String name) {
+ super(name);
+ }
+
+ private static long delegationTokenRemovalScanInterval = 3600000L;
+
+ private AbstractDelegationTokenSecretManager
+ <TimelineDelegationTokenIdentifier> secretManager = null;
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ long secretKeyInterval =
+ conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL,
+ YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL);
+ long tokenMaxLifetime =
+ conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME,
+ YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME);
+ long tokenRenewInterval =
+ conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL,
+ YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL);
+ secretManager = createTimelineDelegationTokenSecretManager(
+ secretKeyInterval, tokenMaxLifetime, tokenRenewInterval,
+ delegationTokenRemovalScanInterval);
+ super.init(conf);
+ }
+
+ protected abstract
+ AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier>
+ createTimelineDelegationTokenSecretManager(long secretKeyInterval,
+ long tokenMaxLifetime, long tokenRenewInterval,
+ long tokenRemovalScanInterval);
+
+ @Override
+ protected void serviceStart() throws Exception {
+ secretManager.startThreads();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ secretManager.stopThreads();
+ super.stop();
+ }
+
+ public AbstractDelegationTokenSecretManager
+ <TimelineDelegationTokenIdentifier>
+ getTimelineDelegationTokenSecretManager() {
+ return secretManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java
new file mode 100644
index 0000000..14a52e3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.server.timeline.security contains classes related
+ * to timeline authentication filters and abstract delegation token service for
+ * ATSv1 and ATSv2.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.server.timeline.security;
+import org.apache.hadoop.classification.InterfaceAudience;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org