You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2017/08/30 20:16:03 UTC
[38/50] [abbrv] hadoop git commit: YARN-7006. [ATSv2 Security]
Changes for authentication for CollectorNodemanagerProtocol. Contributed by
Varun Saxena
YARN-7006. [ATSv2 Security] Changes for authentication for CollectorNodemanagerProtocol. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7c87c90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7c87c90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7c87c90
Branch: refs/heads/YARN-5355_branch2
Commit: c7c87c90d0c4e756d399cde41b9f2e7ff0f979e1
Parents: 6ea488b
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 16 11:01:06 2017 -0700
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Aug 31 01:41:58 2017 +0530
----------------------------------------------------------------------
.../collectormanager/NMCollectorService.java | 7 +-
.../containermanager/AuxServices.java | 3 +-
.../timelineservice/NMTimelinePublisher.java | 29 ++++++--
.../CollectorNodemanagerSecurityInfo.java | 69 ++++++++++++++++++++
.../org.apache.hadoop.security.SecurityInfo | 14 ++++
5 files changed, 112 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c87c90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index 7fdca78..862cd1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -72,13 +72,13 @@ public class NMCollectorService extends CompositeService implements
Configuration serverConf = new Configuration(conf);
- // TODO Security settings.
YarnRPC rpc = YarnRPC.create(conf);
+ // Kerberos based authentication to be used for CollectorNodemanager
+ // protocol if security is enabled.
server =
rpc.getServer(CollectorNodemanagerProtocol.class, this,
- collectorServerAddress, serverConf,
- this.context.getNMTokenSecretManager(),
+ collectorServerAddress, serverConf, null,
conf.getInt(YarnConfiguration.NM_COLLECTOR_SERVICE_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT));
@@ -93,7 +93,6 @@ public class NMCollectorService extends CompositeService implements
LOG.info("NMCollectorService started at " + collectorServerAddress);
}
-
@Override
public void serviceStop() throws Exception {
if (server != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c87c90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index 2efc932..5e0f293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -244,7 +244,8 @@ public class AuxServices extends AbstractService
for (AuxiliaryService serv : serviceMap.values()) {
try {
serv.initializeContainer(new ContainerInitializationContext(
- event.getUser(), event.getContainer().getContainerId(),
+ event.getContainer().getUser(),
+ event.getContainer().getContainerId(),
event.getContainer().getResource(), event.getContainer()
.getContainerTokenIdentifier().getContainerType()));
} catch (Throwable th) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c87c90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index ced41c5..ee3ab02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -26,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -77,6 +79,8 @@ public class NMTimelinePublisher extends CompositeService {
private String httpAddress;
+ private UserGroupInformation nmLoginUGI;
+
private final Map<ApplicationId, TimelineV2Client> appToClientMap;
public NMTimelinePublisher(Context context) {
@@ -91,6 +95,9 @@ public class NMTimelinePublisher extends CompositeService {
dispatcher.register(NMTimelineEventType.class,
new ForwardingEventHandler());
addIfService(dispatcher);
+ this.nmLoginUGI = UserGroupInformation.isSecurityEnabled() ?
+ UserGroupInformation.getLoginUser() :
+ UserGroupInformation.getCurrentUser();
super.serviceInit(conf);
}
@@ -398,11 +405,23 @@ public class NMTimelinePublisher extends CompositeService {
public void createTimelineClient(ApplicationId appId) {
if (!appToClientMap.containsKey(appId)) {
- TimelineV2Client timelineClient =
- TimelineV2Client.createTimelineClient(appId);
- timelineClient.init(getConfig());
- timelineClient.start();
- appToClientMap.put(appId, timelineClient);
+ try {
+ TimelineV2Client timelineClient =
+ nmLoginUGI.doAs(new PrivilegedExceptionAction<TimelineV2Client>() {
+ @Override
+ public TimelineV2Client run() throws Exception {
+ TimelineV2Client timelineClient =
+ TimelineV2Client.createTimelineClient(appId);
+ timelineClient.init(getConfig());
+ timelineClient.start();
+ return timelineClient;
+ }
+ });
+ appToClientMap.put(appId, timelineClient);
+ } catch (IOException | InterruptedException | RuntimeException |
+ Error e) {
+ LOG.warn("Unable to create timeline client for app " + appId, e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c87c90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java
new file mode 100644
index 0000000..0eb5ee5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/security/CollectorNodemanagerSecurityInfo.java
@@ -0,0 +1,69 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.timelineservice.security;
+
+import java.lang.annotation.Annotation;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
+
+/**
+ * SecurityInfo implementation for CollectorNodemanager protocol.
+ */
+@Public
+@Evolving
+public class CollectorNodemanagerSecurityInfo extends SecurityInfo {
+
+ @Override
+ public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
+ if (!protocol
+ .equals(CollectorNodemanagerProtocolPB.class)) {
+ return null;
+ }
+ return new KerberosInfo() {
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return null;
+ }
+
+ @Override
+ public String serverPrincipal() {
+ return YarnConfiguration.NM_PRINCIPAL;
+ }
+
+ @Override
+ public String clientPrincipal() {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
+ return null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c87c90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
new file mode 100644
index 0000000..4389219
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.yarn.server.timelineservice.security.CollectorNodemanagerSecurityInfo
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org