You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/12/07 00:54:17 UTC
[32/50] [abbrv] hadoop git commit: YARN-6669. Implemented Kerberos
security for YARN service framework. (Contributed by Jian He)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
index e18bcae..173001b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.DNS;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -43,9 +45,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.net.URLDecoder;
+import java.net.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -57,6 +57,11 @@ import java.util.Map;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic
+ .HADOOP_SECURITY_DNS_INTERFACE_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic
+ .HADOOP_SECURITY_DNS_NAMESERVER_KEY;
+
/**
* These are slider-specific Util methods
*/
@@ -542,4 +547,24 @@ public final class ServiceUtils {
public static String createDescriptionTag(String description) {
return "Description: " + description;
}
+
+ // Copied from SecurityUtil because it is not public
+ public static String getLocalHostName(@Nullable Configuration conf)
+ throws UnknownHostException {
+ if (conf != null) {
+ String dnsInterface = conf.get(HADOOP_SECURITY_DNS_INTERFACE_KEY);
+ String nameServer = conf.get(HADOOP_SECURITY_DNS_NAMESERVER_KEY);
+
+ if (dnsInterface != null) {
+ return DNS.getDefaultHost(dnsInterface, nameServer, true);
+ } else if (nameServer != null) {
+ throw new IllegalArgumentException(HADOOP_SECURITY_DNS_NAMESERVER_KEY +
+ " requires " + HADOOP_SECURITY_DNS_INTERFACE_KEY + ". Check your" +
+ "configuration.");
+ }
+ }
+
+ // Fallback to querying the default hostname as we did before.
+ return InetAddress.getLocalHost().getCanonicalHostName();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
new file mode 100644
index 0000000..14cdf68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.yarn.service.ClientAMSecurityInfo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
index 704b097..e74ca81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
@@ -111,6 +111,27 @@ public final class RegistryOperationsFactory {
}
/**
+ * Create a kerberos registry service client
+ * @param conf configuration
+ * @param jaasClientEntry the name of the login config entry
+ * @param principal principal of the client.
+ * @param keytab location to the keytab file
+ * @return a registry service client instance
+ */
+ public static RegistryOperations createKerberosInstance(Configuration conf,
+ String jaasClientEntry, String principal, String keytab) {
+ Preconditions.checkArgument(conf != null, "Null configuration");
+ conf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_KERBEROS);
+ conf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, jaasClientEntry);
+ RegistryOperationsClient operations =
+ new RegistryOperationsClient("KerberosRegistryOperations");
+ operations.setKerberosPrincipalAndKeytab(principal, keytab);
+ operations.init(conf);
+ return operations;
+ }
+
+
+ /**
* Create and initialize an operations instance authenticated with write
* access via an <code>id:password</code> pair.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index 8713920..c81a0ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -127,6 +127,7 @@ public class CuratorService extends CompositeService
} else {
this.bindingSource = this;
}
+ registrySecurity = new RegistrySecurity("registry security");
}
/**
@@ -152,8 +153,7 @@ public class CuratorService extends CompositeService
registryRoot = conf.getTrimmed(KEY_REGISTRY_ZK_ROOT,
DEFAULT_ZK_REGISTRY_ROOT);
- // create and add the registy service
- registrySecurity = new RegistrySecurity("registry security");
+ // add the registy service
addService(registrySecurity);
if (LOG.isDebugEnabled()) {
@@ -163,6 +163,10 @@ public class CuratorService extends CompositeService
super.serviceInit(conf);
}
+ public void setKerberosPrincipalAndKeytab(String principal, String keytab) {
+ registrySecurity.setKerberosPrincipalAndKeytab(principal, keytab);
+ }
+
/**
* Start the service.
* This is where the curator instance is started.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 23fadb5..ff6e8aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -23,6 +23,7 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
@@ -31,6 +32,7 @@ import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.util.ZKUtil;
import org.apache.zookeeper.Environment;
import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
@@ -44,9 +46,11 @@ import java.lang.reflect.InvocationTargetException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
+import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import static org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions.*;
@@ -170,13 +174,17 @@ public class RegistrySecurity extends AbstractService {
/**
* Client context
*/
- private String jaasClientContext;
+ private String jaasClientEntry;
/**
* Client identity
*/
private String jaasClientIdentity;
+ private String principal;
+
+ private String keytab;
+
/**
* Create an instance
* @param name service name
@@ -238,6 +246,8 @@ public class RegistrySecurity extends AbstractService {
systemACLs.addAll(buildACLs(system, kerberosRealm, ZooDefs.Perms.ALL));
+ LOG.info("Registry default system acls: " + System.lineSeparator() +
+ systemACLs);
// user accounts (may be empty, but for digest one user AC must
// be built up
String user = getConfig().get(KEY_REGISTRY_USER_ACCOUNTS,
@@ -252,6 +262,7 @@ public class RegistrySecurity extends AbstractService {
userACLs.add(self);
}
}
+ LOG.info("Registry User ACLs " + System.lineSeparator()+ userACLs);
// here check for UGI having secure on or digest + ID
switch (access) {
@@ -262,13 +273,12 @@ public class RegistrySecurity extends AbstractService {
}
UserGroupInformation currentUser =
UserGroupInformation.getCurrentUser();
- jaasClientContext = getOrFail(KEY_REGISTRY_CLIENT_JAAS_CONTEXT,
+ jaasClientEntry = getOrFail(KEY_REGISTRY_CLIENT_JAAS_CONTEXT,
DEFAULT_REGISTRY_CLIENT_JAAS_CONTEXT);
jaasClientIdentity = currentUser.getShortUserName();
if (LOG.isDebugEnabled()) {
LOG.debug("Auth is SASL user=\"{}\" JAAS context=\"{}\"",
- jaasClientIdentity,
- jaasClientContext);
+ jaasClientIdentity, jaasClientEntry);
}
break;
@@ -738,9 +748,81 @@ public class RegistrySecurity extends AbstractService {
break;
case sasl:
- // bind to the current identity and context within the JAAS file
- setZKSaslClientProperties(jaasClientIdentity, jaasClientContext);
+ JaasConfiguration jconf =
+ new JaasConfiguration(jaasClientEntry, principal, keytab);
+ javax.security.auth.login.Configuration.setConfiguration(jconf);
+ setSystemPropertyIfUnset(ZooKeeperSaslClient.ENABLE_CLIENT_SASL_KEY,
+ "true");
+ setSystemPropertyIfUnset(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+ jaasClientEntry);
+ LOG.info(
+ "Enabling ZK sasl client: jaasClientEntry = " + jaasClientEntry
+ + ", principal = " + principal + ", keytab = " + keytab);
+ }
+ }
+ }
+
+ public void setKerberosPrincipalAndKeytab(String principal, String keytab) {
+ this.principal = principal;
+ this.keytab = keytab;
+ }
+
+ /**
+ * Creates a programmatic version of a jaas.conf file. This can be used
+ * instead of writing a jaas.conf file and setting the system property,
+ * "java.security.auth.login.config", to point to that file. It is meant to be
+ * used for connecting to ZooKeeper.
+ */
+ @InterfaceAudience.Private
+ public static class JaasConfiguration extends
+ javax.security.auth.login.Configuration {
+
+ private final javax.security.auth.login.Configuration baseConfig =
+ javax.security.auth.login.Configuration.getConfiguration();
+ private static AppConfigurationEntry[] entry;
+ private String entryName;
+
+ /**
+ * Add an entry to the jaas configuration with the passed in name,
+ * principal, and keytab. The other necessary options will be set for you.
+ *
+ * @param entryName The name of the entry (e.g. "Client")
+ * @param principal The principal of the user
+ * @param keytab The location of the keytab
+ */
+ public JaasConfiguration(String entryName, String principal, String keytab) {
+ this.entryName = entryName;
+ Map<String, String> options = new HashMap<String, String>();
+ options.put("keyTab", keytab);
+ options.put("principal", principal);
+ options.put("useKeyTab", "true");
+ options.put("storeKey", "true");
+ options.put("useTicketCache", "false");
+ options.put("refreshKrb5Config", "true");
+ String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
+ if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
+ options.put("debug", "true");
+ }
+ entry = new AppConfigurationEntry[]{
+ new AppConfigurationEntry(getKrb5LoginModuleName(),
+ AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+ options)};
+ }
+
+ @Override
+ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+ return (entryName.equals(name)) ? entry : ((baseConfig != null)
+ ? baseConfig.getAppConfigurationEntry(name) : null);
+ }
+
+ private String getKrb5LoginModuleName() {
+ String krb5LoginModuleName;
+ if (System.getProperty("java.vendor").contains("IBM")) {
+ krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
+ } else {
+ krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}
+ return krb5LoginModuleName;
}
}
@@ -899,7 +981,7 @@ public class RegistrySecurity extends AbstractService {
.append("; ");
builder.append(KEY_REGISTRY_CLIENT_JAAS_CONTEXT)
.append("=")
- .append(jaasClientContext)
+ .append(jaasClientEntry)
.append("; ");
builder.append(describeProperty(PROP_ZK_SASL_CLIENT_USERNAME));
builder.append(describeProperty(PROP_ZK_SASL_CLIENT_CONTEXT));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index d7ebece..358a963 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -412,6 +412,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
// Single reverse zone
} else {
Name reverseLookupZoneName = getReverseZoneName(conf);
+ if (reverseLookupZoneName == null) {
+ // reverse lookup disabled
+ return;
+ }
Zone reverseLookupZone = configureZone(reverseLookupZoneName, conf);
zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java
deleted file mode 100644
index e11890f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.registry.server.integration;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.curator.framework.api.BackgroundCallback;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.registry.client.impl.zk.RegistryBindingSource;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.server.services.DeleteCompletionCallback;
-import org.apache.hadoop.registry.server.services.RegistryAdminService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Future;
-
-/**
- * Handle RM events by updating the registry
- * <p>
- * These actions are all implemented as event handlers to operations
- * which come from the RM.
- * <p>
- * This service is expected to be executed by a user with the permissions
- * to manipulate the entire registry,
- */
-@InterfaceAudience.LimitedPrivate("YARN")
-@InterfaceStability.Evolving
-public class RMRegistryOperationsService extends RegistryAdminService {
- private static final Logger LOG =
- LoggerFactory.getLogger(RMRegistryOperationsService.class);
-
- private PurgePolicy purgeOnCompletionPolicy = PurgePolicy.PurgeAll;
-
- public RMRegistryOperationsService(String name) {
- this(name, null);
- }
-
- public RMRegistryOperationsService(String name,
- RegistryBindingSource bindingSource) {
- super(name, bindingSource);
- }
-
-
- /**
- * Extend the parent service initialization by verifying that the
- * service knows —in a secure cluster— the realm in which it is executing.
- * It needs this to properly build up the user names and hence their
- * access rights.
- *
- * @param conf configuration of the service
- * @throws Exception
- */
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
-
- verifyRealmValidity();
- }
-
- public PurgePolicy getPurgeOnCompletionPolicy() {
- return purgeOnCompletionPolicy;
- }
-
- public void setPurgeOnCompletionPolicy(PurgePolicy purgeOnCompletionPolicy) {
- this.purgeOnCompletionPolicy = purgeOnCompletionPolicy;
- }
-
- public void onApplicationAttemptRegistered(ApplicationAttemptId attemptId,
- String host, int rpcport, String trackingurl) throws IOException {
-
- }
-
- public void onApplicationLaunched(ApplicationId id) throws IOException {
-
- }
-
- /**
- * Actions to take as an AM registers itself with the RM.
- * @param attemptId attempt ID
- * @throws IOException problems
- */
- public void onApplicationMasterRegistered(ApplicationAttemptId attemptId) throws
- IOException {
- }
-
- /**
- * Actions to take when the AM container is completed
- * @param containerId container ID
- * @throws IOException problems
- */
- public void onAMContainerFinished(ContainerId containerId) throws
- IOException {
- LOG.info("AM Container {} finished, purging application attempt records",
- containerId);
-
- // remove all application attempt entries
- purgeAppAttemptRecords(containerId.getApplicationAttemptId());
-
- // also treat as a container finish to remove container
- // level records for the AM container
- onContainerFinished(containerId);
- }
-
- /**
- * remove all application attempt entries
- * @param attemptId attempt ID
- */
- protected void purgeAppAttemptRecords(ApplicationAttemptId attemptId) {
- purgeRecordsAsync("/",
- attemptId.toString(),
- PersistencePolicies.APPLICATION_ATTEMPT);
- }
-
- /**
- * Actions to take when an application attempt is completed
- * @param attemptId application ID
- * @throws IOException problems
- */
- public void onApplicationAttemptUnregistered(ApplicationAttemptId attemptId)
- throws IOException {
- LOG.info("Application attempt {} unregistered, purging app attempt records",
- attemptId);
- purgeAppAttemptRecords(attemptId);
- }
-
- /**
- * Actions to take when an application is completed
- * @param id application ID
- * @throws IOException problems
- */
- public void onApplicationCompleted(ApplicationId id)
- throws IOException {
- LOG.info("Application {} completed, purging application-level records",
- id);
- purgeRecordsAsync("/",
- id.toString(),
- PersistencePolicies.APPLICATION);
- }
-
- public void onApplicationAttemptAdded(ApplicationAttemptId appAttemptId) {
- }
-
- /**
- * This is the event where the user is known, so the user directory
- * can be created
- * @param applicationId application ID
- * @param user username
- * @throws IOException problems
- */
- public void onStateStoreEvent(ApplicationId applicationId, String user) throws
- IOException {
- initUserRegistryAsync(user);
- }
-
- /**
- * Actions to take when the AM container is completed
- * @param id container ID
- * @throws IOException problems
- */
- public void onContainerFinished(ContainerId id) throws IOException {
- LOG.info("Container {} finished, purging container-level records",
- id);
- purgeRecordsAsync("/",
- id.toString(),
- PersistencePolicies.CONTAINER);
- }
-
- /**
- * Queue an async operation to purge all matching records under a base path.
- * <ol>
- * <li>Uses a depth first search</li>
- * <li>A match is on ID and persistence policy, or, if policy==-1, any match</li>
- * <li>If a record matches then it is deleted without any child searches</li>
- * <li>Deletions will be asynchronous if a callback is provided</li>
- * </ol>
- * @param path base path
- * @param id ID for service record.id
- * @param persistencePolicyMatch ID for the persistence policy to match:
- * no match, no delete.
- * @return a future that returns the #of records deleted
- */
- @VisibleForTesting
- public Future<Integer> purgeRecordsAsync(String path,
- String id,
- String persistencePolicyMatch) {
-
- return purgeRecordsAsync(path,
- id, persistencePolicyMatch,
- purgeOnCompletionPolicy,
- new DeleteCompletionCallback());
- }
-
- /**
- * Queue an async operation to purge all matching records under a base path.
- * <ol>
- * <li>Uses a depth first search</li>
- * <li>A match is on ID and persistence policy, or, if policy==-1, any match</li>
- * <li>If a record matches then it is deleted without any child searches</li>
- * <li>Deletions will be asynchronous if a callback is provided</li>
- * </ol>
- * @param path base path
- * @param id ID for service record.id
- * @param persistencePolicyMatch ID for the persistence policy to match:
- * no match, no delete.
- * @param purgePolicy how to react to children under the entry
- * @param callback an optional callback
- * @return a future that returns the #of records deleted
- */
- @VisibleForTesting
- public Future<Integer> purgeRecordsAsync(String path,
- String id,
- String persistencePolicyMatch,
- PurgePolicy purgePolicy,
- BackgroundCallback callback) {
- LOG.info(" records under {} with ID {} and policy {}: {}",
- path, id, persistencePolicyMatch);
- return submit(
- new AsyncPurge(path,
- new SelectByYarnPersistence(id, persistencePolicyMatch),
- purgePolicy,
- callback));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
index e160d4a..829ef68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.registry.server.services;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.BackgroundCallback;
import org.apache.curator.framework.api.CuratorEvent;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class DeleteCompletionCallback implements BackgroundCallback {
private static final Logger LOG =
- LoggerFactory.getLogger(RMRegistryOperationsService.class);
+ LoggerFactory.getLogger(DeleteCompletionCallback.class);
private AtomicInteger events = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
index 5b34f60..0d4a467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java
@@ -23,7 +23,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
+
+import org.apache.hadoop.registry.server.services.RegistryAdminService;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,22 +32,16 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URISyntaxException;
-/**
- * Abstract registry tests .. inits the field {@link #registry}
- * before the test with an instance of {@link RMRegistryOperationsService};
- * and {@link #operations} with the same instance cast purely
- * to the type {@link RegistryOperations}.
- *
- */
+
public class AbstractRegistryTest extends AbstractZKRegistryTest {
private static final Logger LOG =
LoggerFactory.getLogger(AbstractRegistryTest.class);
- protected RMRegistryOperationsService registry;
+ protected RegistryAdminService registry;
protected RegistryOperations operations;
@Before
public void setupRegistry() throws IOException {
- registry = new RMRegistryOperationsService("yarnRegistry");
+ registry = new RegistryAdminService("yarnRegistry");
operations = registry;
registry.init(createRegistryConfiguration());
registry.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
deleted file mode 100644
index 451a69b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
+++ /dev/null
@@ -1,369 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.registry.integration;
-
-import org.apache.curator.framework.api.BackgroundCallback;
-import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
-import org.apache.hadoop.registry.AbstractRegistryTest;
-import org.apache.hadoop.registry.client.api.BindFlags;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
-import org.apache.hadoop.registry.client.impl.CuratorEventCatcher;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.RegistryPathStatus;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.registry.server.services.DeleteCompletionCallback;
-import org.apache.hadoop.registry.server.services.RegistryAdminService;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.inetAddrEndpoint;
-import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.restEndpoint;
-
-public class TestRegistryRMOperations extends AbstractRegistryTest {
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestRegistryRMOperations.class);
-
- /**
- * trigger a purge operation
- * @param path path
- * @param id yarn ID
- * @param policyMatch policy to match ID on
- * @param purgePolicy policy when there are children under a match
- * @return the number purged
- * @throws IOException
- */
- public int purge(String path,
- String id,
- String policyMatch,
- RegistryAdminService.PurgePolicy purgePolicy) throws
- IOException,
- ExecutionException,
- InterruptedException {
- return purge(path, id, policyMatch, purgePolicy, null);
- }
-
- /**
- *
- * trigger a purge operation
- * @param path pathn
- * @param id yarn ID
- * @param policyMatch policy to match ID on
- * @param purgePolicy policy when there are children under a match
- * @param callback optional callback
- * @return the number purged
- * @throws IOException
- */
- public int purge(String path,
- String id,
- String policyMatch,
- RegistryAdminService.PurgePolicy purgePolicy,
- BackgroundCallback callback) throws
- IOException,
- ExecutionException,
- InterruptedException {
-
- Future<Integer> future = registry.purgeRecordsAsync(path,
- id, policyMatch, purgePolicy, callback);
- try {
- return future.get();
- } catch (ExecutionException e) {
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- } else {
- throw e;
- }
- }
- }
-
- @Test
- public void testPurgeEntryCuratorCallback() throws Throwable {
-
- String path = "/users/example/hbase/hbase1/";
- ServiceRecord written = buildExampleServiceEntry(
- PersistencePolicies.APPLICATION_ATTEMPT);
- written.set(YarnRegistryAttributes.YARN_ID,
- "testAsyncPurgeEntry_attempt_001");
-
- operations.mknode(RegistryPathUtils.parentOf(path), true);
- operations.bind(path, written, 0);
-
- ZKPathDumper dump = registry.dumpPath(false);
- CuratorEventCatcher events = new CuratorEventCatcher();
-
- LOG.info("Initial state {}", dump);
-
- // container query
- String id = written.get(YarnRegistryAttributes.YARN_ID, "");
- int opcount = purge("/",
- id,
- PersistencePolicies.CONTAINER,
- RegistryAdminService.PurgePolicy.PurgeAll,
- events);
- assertPathExists(path);
- assertEquals(0, opcount);
- assertEquals("Event counter", 0, events.getCount());
-
- // now the application attempt
- opcount = purge("/",
- id,
- PersistencePolicies.APPLICATION_ATTEMPT,
- RegistryAdminService.PurgePolicy.PurgeAll,
- events);
-
- LOG.info("Final state {}", dump);
-
- assertPathNotFound(path);
- assertEquals("wrong no of delete operations in " + dump, 1, opcount);
- // and validate the callback event
- assertEquals("Event counter", 1, events.getCount());
- }
-
- @Test
- public void testAsyncPurgeEntry() throws Throwable {
-
- String path = "/users/example/hbase/hbase1/";
- ServiceRecord written = buildExampleServiceEntry(
- PersistencePolicies.APPLICATION_ATTEMPT);
- written.set(YarnRegistryAttributes.YARN_ID,
- "testAsyncPurgeEntry_attempt_001");
-
- operations.mknode(RegistryPathUtils.parentOf(path), true);
- operations.bind(path, written, 0);
-
- ZKPathDumper dump = registry.dumpPath(false);
-
- LOG.info("Initial state {}", dump);
-
- DeleteCompletionCallback deletions = new DeleteCompletionCallback();
- int opcount = purge("/",
- written.get(YarnRegistryAttributes.YARN_ID, ""),
- PersistencePolicies.CONTAINER,
- RegistryAdminService.PurgePolicy.PurgeAll,
- deletions);
- assertPathExists(path);
-
- dump = registry.dumpPath(false);
-
- assertEquals("wrong no of delete operations in " + dump, 0,
- deletions.getEventCount());
- assertEquals("wrong no of delete operations in " + dump, 0, opcount);
-
-
- // now app attempt
- deletions = new DeleteCompletionCallback();
- opcount = purge("/",
- written.get(YarnRegistryAttributes.YARN_ID, ""),
- PersistencePolicies.APPLICATION_ATTEMPT,
- RegistryAdminService.PurgePolicy.PurgeAll,
- deletions);
-
- dump = registry.dumpPath(false);
- LOG.info("Final state {}", dump);
-
- assertPathNotFound(path);
- assertEquals("wrong no of delete operations in " + dump, 1,
- deletions.getEventCount());
- assertEquals("wrong no of delete operations in " + dump, 1, opcount);
- // and validate the callback event
-
- }
-
- @Test
- public void testPutGetContainerPersistenceServiceEntry() throws Throwable {
-
- String path = ENTRY_PATH;
- ServiceRecord written = buildExampleServiceEntry(
- PersistencePolicies.CONTAINER);
-
- operations.mknode(RegistryPathUtils.parentOf(path), true);
- operations.bind(path, written, BindFlags.CREATE);
- ServiceRecord resolved = operations.resolve(path);
- validateEntry(resolved);
- assertMatches(written, resolved);
- }
-
- /**
- * Create a complex example app
- * @throws Throwable
- */
- @Test
- public void testCreateComplexApplication() throws Throwable {
- String appId = "application_1408631738011_0001";
- String cid = "container_1408631738011_0001_01_";
- String cid1 = cid + "000001";
- String cid2 = cid + "000002";
- String appPath = USERPATH + "tomcat";
-
- ServiceRecord webapp = createRecord(appId,
- PersistencePolicies.APPLICATION, "tomcat-based web application",
- null);
- webapp.addExternalEndpoint(restEndpoint("www",
- new URI("http", "//loadbalancer/", null)));
-
- ServiceRecord comp1 = createRecord(cid1, PersistencePolicies.CONTAINER,
- null,
- null);
- comp1.addExternalEndpoint(restEndpoint("www",
- new URI("http", "//rack4server3:43572", null)));
- comp1.addInternalEndpoint(
- inetAddrEndpoint("jmx", "JMX", "rack4server3", 43573));
-
- // Component 2 has a container lifespan
- ServiceRecord comp2 = createRecord(cid2, PersistencePolicies.CONTAINER,
- null,
- null);
- comp2.addExternalEndpoint(restEndpoint("www",
- new URI("http", "//rack1server28:35881", null)));
- comp2.addInternalEndpoint(
- inetAddrEndpoint("jmx", "JMX", "rack1server28", 35882));
-
- operations.mknode(USERPATH, false);
- operations.bind(appPath, webapp, BindFlags.OVERWRITE);
- String componentsPath = appPath + RegistryConstants.SUBPATH_COMPONENTS;
- operations.mknode(componentsPath, false);
- String dns1 = RegistryPathUtils.encodeYarnID(cid1);
- String dns1path = componentsPath + dns1;
- operations.bind(dns1path, comp1, BindFlags.CREATE);
- String dns2 = RegistryPathUtils.encodeYarnID(cid2);
- String dns2path = componentsPath + dns2;
- operations.bind(dns2path, comp2, BindFlags.CREATE);
-
- ZKPathDumper pathDumper = registry.dumpPath(false);
- LOG.info(pathDumper.toString());
-
- logRecord("tomcat", webapp);
- logRecord(dns1, comp1);
- logRecord(dns2, comp2);
-
- ServiceRecord dns1resolved = operations.resolve(dns1path);
- assertEquals("Persistence policies on resolved entry",
- PersistencePolicies.CONTAINER,
- dns1resolved.get(YarnRegistryAttributes.YARN_PERSISTENCE, ""));
-
- Map<String, RegistryPathStatus> children =
- RegistryUtils.statChildren(operations, componentsPath);
- assertEquals(2, children.size());
- Collection<RegistryPathStatus>
- componentStats = children.values();
- Map<String, ServiceRecord> records =
- RegistryUtils.extractServiceRecords(operations,
- componentsPath, componentStats);
- assertEquals(2, records.size());
- ServiceRecord retrieved1 = records.get(dns1path);
- logRecord(retrieved1.get(YarnRegistryAttributes.YARN_ID, ""), retrieved1);
- assertMatches(dns1resolved, retrieved1);
- assertEquals(PersistencePolicies.CONTAINER,
- retrieved1.get(YarnRegistryAttributes.YARN_PERSISTENCE, ""));
-
- // create a listing under components/
- operations.mknode(componentsPath + "subdir", false);
-
- // this shows up in the listing of child entries
- Map<String, RegistryPathStatus> childrenUpdated =
- RegistryUtils.statChildren(operations, componentsPath);
- assertEquals(3, childrenUpdated.size());
-
- // the non-record child this is not picked up in the record listing
- Map<String, ServiceRecord> recordsUpdated =
-
- RegistryUtils.extractServiceRecords(operations,
- componentsPath,
- childrenUpdated);
- assertEquals(2, recordsUpdated.size());
-
- // now do some deletions.
-
- // synchronous delete container ID 2
-
- // fail if the app policy is chosen
- assertEquals(0, purge("/", cid2, PersistencePolicies.APPLICATION,
- RegistryAdminService.PurgePolicy.FailOnChildren));
- // succeed for container
- assertEquals(1, purge("/", cid2, PersistencePolicies.CONTAINER,
- RegistryAdminService.PurgePolicy.FailOnChildren));
- assertPathNotFound(dns2path);
- assertPathExists(dns1path);
-
- // expect a skip on children to skip
- assertEquals(0,
- purge("/", appId, PersistencePolicies.APPLICATION,
- RegistryAdminService.PurgePolicy.SkipOnChildren));
- assertPathExists(appPath);
- assertPathExists(dns1path);
-
- // attempt to delete app with policy of fail on children
- try {
- int p = purge("/",
- appId,
- PersistencePolicies.APPLICATION,
- RegistryAdminService.PurgePolicy.FailOnChildren);
- fail("expected a failure, got a purge count of " + p);
- } catch (PathIsNotEmptyDirectoryException expected) {
- // expected
- }
- assertPathExists(appPath);
- assertPathExists(dns1path);
-
-
- // now trigger recursive delete
- assertEquals(1,
- purge("/", appId, PersistencePolicies.APPLICATION,
- RegistryAdminService.PurgePolicy.PurgeAll));
- assertPathNotFound(appPath);
- assertPathNotFound(dns1path);
-
- }
-
- @Test
- public void testChildDeletion() throws Throwable {
- ServiceRecord app = createRecord("app1",
- PersistencePolicies.APPLICATION, "app",
- null);
- ServiceRecord container = createRecord("container1",
- PersistencePolicies.CONTAINER, "container",
- null);
-
- operations.bind("/app", app, BindFlags.OVERWRITE);
- operations.bind("/app/container", container, BindFlags.OVERWRITE);
-
- try {
- int p = purge("/",
- "app1",
- PersistencePolicies.APPLICATION,
- RegistryAdminService.PurgePolicy.FailOnChildren);
- fail("expected a failure, got a purge count of " + p);
- } catch (PathIsNotEmptyDirectoryException expected) {
- // expected
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
deleted file mode 100644
index 41760d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.registry.secure;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.PathPermissionException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.ServiceStateException;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
-import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
-import org.apache.hadoop.registry.client.impl.RegistryOperationsClient;
-import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
-import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
-import org.apache.hadoop.registry.server.services.RegistryAdminService;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.security.auth.login.LoginException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.List;
-
-import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
-
-/**
- * Verify that the {@link RMRegistryOperationsService} works securely
- */
-public class TestSecureRMRegistryOperations extends AbstractSecureRegistryTest {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestSecureRMRegistryOperations.class);
- private Configuration secureConf;
- private Configuration zkClientConf;
- private UserGroupInformation zookeeperUGI;
-
- @Before
- public void setupTestSecureRMRegistryOperations() throws Exception {
- startSecureZK();
- secureConf = new Configuration();
- secureConf.setBoolean(KEY_REGISTRY_SECURE, true);
-
- // create client conf containing the ZK quorum
- zkClientConf = new Configuration(secureZK.getConfig());
- zkClientConf.setBoolean(KEY_REGISTRY_SECURE, true);
- assertNotEmpty(zkClientConf.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM));
-
- // ZK is in charge
- secureConf.set(KEY_REGISTRY_SYSTEM_ACCOUNTS, "sasl:zookeeper@");
- zookeeperUGI = loginUGI(ZOOKEEPER, keytab_zk);
- }
-
- @After
- public void teardownTestSecureRMRegistryOperations() {
- }
-
- /**
- * Create the RM registry operations as the current user
- * @return the service
- * @throws LoginException
- * @throws FileNotFoundException
- */
- public RMRegistryOperationsService startRMRegistryOperations() throws
- LoginException, IOException, InterruptedException {
- // kerberos
- secureConf.set(KEY_REGISTRY_CLIENT_AUTH,
- REGISTRY_CLIENT_AUTH_KERBEROS);
- secureConf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, ZOOKEEPER_CLIENT_CONTEXT);
-
- RMRegistryOperationsService registryOperations = zookeeperUGI.doAs(
- new PrivilegedExceptionAction<RMRegistryOperationsService>() {
- @Override
- public RMRegistryOperationsService run() throws Exception {
- RMRegistryOperationsService operations
- = new RMRegistryOperationsService("rmregistry", secureZK);
- addToTeardown(operations);
- operations.init(secureConf);
- LOG.info(operations.bindingDiagnosticDetails());
- operations.start();
- return operations;
- }
- });
-
- return registryOperations;
- }
-
- /**
- * test that ZK can write as itself
- * @throws Throwable
- */
- @Test
- public void testZookeeperCanWriteUnderSystem() throws Throwable {
-
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- RegistryOperations operations = rmRegistryOperations;
- operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
- false);
- ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
- LOG.info(pathDumper.toString());
- }
-
- @Test
- public void testAnonReadAccess() throws Throwable {
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- describe(LOG, "testAnonReadAccess");
- RegistryOperations operations =
- RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
- addToTeardown(operations);
- operations.start();
-
- assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
- RegistrySecurity.isClientSASLEnabled());
- operations.list(PATH_SYSTEM_SERVICES);
- }
-
- @Test
- public void testAnonNoWriteAccess() throws Throwable {
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- describe(LOG, "testAnonNoWriteAccess");
- RegistryOperations operations =
- RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
- addToTeardown(operations);
- operations.start();
-
- String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
- expectMkNodeFailure(operations, servicePath);
- }
-
- @Test
- public void testAnonNoWriteAccessOffRoot() throws Throwable {
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- describe(LOG, "testAnonNoWriteAccessOffRoot");
- RegistryOperations operations =
- RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
- addToTeardown(operations);
- operations.start();
- assertFalse("mknode(/)", operations.mknode("/", false));
- expectMkNodeFailure(operations, "/sub");
- expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
- }
-
- /**
- * Expect a mknode operation to fail
- * @param operations operations instance
- * @param path path
- * @throws IOException An IO failure other than those permitted
- */
- public void expectMkNodeFailure(RegistryOperations operations,
- String path) throws IOException {
- try {
- operations.mknode(path, false);
- fail("should have failed to create a node under " + path);
- } catch (PathPermissionException expected) {
- // expected
- } catch (NoPathPermissionsException expected) {
- // expected
- }
- }
-
- /**
- * Expect a delete operation to fail
- * @param operations operations instance
- * @param path path
- * @param recursive
- * @throws IOException An IO failure other than those permitted
- */
- public void expectDeleteFailure(RegistryOperations operations,
- String path, boolean recursive) throws IOException {
- try {
- operations.delete(path, recursive);
- fail("should have failed to delete the node " + path);
- } catch (PathPermissionException expected) {
- // expected
- } catch (NoPathPermissionsException expected) {
- // expected
- }
- }
-
- @Test
- public void testAlicePathRestrictedAnonAccess() throws Throwable {
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
- describe(LOG, "Creating anonymous accessor");
- RegistryOperations anonOperations =
- RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
- addToTeardown(anonOperations);
- anonOperations.start();
- anonOperations.list(aliceHome);
- expectMkNodeFailure(anonOperations, aliceHome + "/anon");
- expectDeleteFailure(anonOperations, aliceHome, true);
- }
-
- @Test
- public void testUserZookeeperHomePathAccess() throws Throwable {
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- final String home = rmRegistryOperations.initUserRegistry(ZOOKEEPER);
- describe(LOG, "Creating ZK client");
-
- RegistryOperations operations = zookeeperUGI.doAs(
- new PrivilegedExceptionAction<RegistryOperations>() {
- @Override
- public RegistryOperations run() throws Exception {
- RegistryOperations operations =
- RegistryOperationsFactory.createKerberosInstance(zkClientConf,
- ZOOKEEPER_CLIENT_CONTEXT);
- addToTeardown(operations);
- operations.start();
-
- return operations;
- }
- });
- operations.list(home);
- String path = home + "/subpath";
- operations.mknode(path, false);
- operations.delete(path, true);
- }
-
- @Test
- public void testUserHomedirsPermissionsRestricted() throws Throwable {
- // test that the /users/$user permissions are restricted
- RMRegistryOperationsService rmRegistryOperations =
- startRMRegistryOperations();
- // create Alice's dir, so it should have an ACL for Alice
- final String home = rmRegistryOperations.initUserRegistry(ALICE);
- List<ACL> acls = rmRegistryOperations.zkGetACLS(home);
- ACL aliceACL = null;
- for (ACL acl : acls) {
- LOG.info(RegistrySecurity.aclToString(acl));
- Id id = acl.getId();
- if (id.getScheme().equals(ZookeeperConfigOptions.SCHEME_SASL)
- && id.getId().startsWith(ALICE)) {
-
- aliceACL = acl;
- break;
- }
- }
- assertNotNull(aliceACL);
- assertEquals(RegistryAdminService.USER_HOMEDIR_ACL_PERMISSIONS,
- aliceACL.getPerms());
- }
-
- @Test
- public void testDigestAccess() throws Throwable {
- RMRegistryOperationsService registryAdmin =
- startRMRegistryOperations();
- String id = "username";
- String pass = "password";
- registryAdmin.addWriteAccessor(id, pass);
- List<ACL> clientAcls = registryAdmin.getClientAcls();
- LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));
-
- String base = "/digested";
- registryAdmin.mknode(base, false);
- List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
- String aclset = RegistrySecurity.aclsToString(baseACLs);
- LOG.info("Base ACLs=\n{}", aclset);
- ACL found = null;
- for (ACL acl : baseACLs) {
- if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
- found = acl;
- break;
- }
- }
- assertNotNull("Did not find digest entry in ACLs " + aclset, found);
- zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
- "sasl:somebody@EXAMPLE.COM, sasl:other");
- RegistryOperations operations =
- RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
- id,
- pass);
- addToTeardown(operations);
- operations.start();
- RegistryOperationsClient operationsClient =
- (RegistryOperationsClient) operations;
- List<ACL> digestClientACLs = operationsClient.getClientAcls();
- LOG.info("digest client ACLs=\n{}",
- RegistrySecurity.aclsToString(digestClientACLs));
- operations.stat(base);
- operations.mknode(base + "/subdir", false);
- ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
- LOG.info(pathDumper.toString());
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testNoDigestAuthMissingId() throws Throwable {
- RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
- "",
- "pass");
- }
-
- @Test(expected = ServiceStateException.class)
- public void testNoDigestAuthMissingId2() throws Throwable {
- zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST);
- zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "");
- zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "pass");
- RegistryOperationsFactory.createInstance("DigestRegistryOperations",
- zkClientConf);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testNoDigestAuthMissingPass() throws Throwable {
- RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
- "id",
- "");
- }
-
- @Test(expected = ServiceStateException.class)
- public void testNoDigestAuthMissingPass2() throws Throwable {
- zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST);
- zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "id");
- zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "");
- RegistryOperationsFactory.createInstance("DigestRegistryOperations",
- zkClientConf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 512c011..f13d7d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -208,7 +208,25 @@ If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn`
</property>
```
-# Try with Docker
+# Run with security
+YARN service framework supports running in a secure(kerberized) environment. User needs to specify the kerberos principal name and keytab when they launch the service.
+E.g. A typical configuration looks like below:
+```
+{
+ "name": "sample-service",
+ ...
+ ...
+ "kerberos_principal" : {
+ "principal_name" : "hdfs-demo@EXAMPLE.COM",
+ "keytab" : "hdfs:///etc/security/keytabs/hdfs.headless.keytab"
+ }
+}
+```
+* principal_name : the principal name of the user who launches the service
+* keytab : URI of the keytab. It supports two modes:
+ * URI starts with `hdfs://`: The URI where the keytab is stored on hdfs. The keytab will be localized to each node by YARN.
+ * URI starts with `file://`: The URI where the keytab is stored on local host. It is assumed that admin pre-installs the keytabs on the local host before AM launches.
+# Run with Docker
The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services.
Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image.
For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.md).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index c0e12c7..e224e5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -291,6 +291,15 @@ The current state of the container of a service.
|----|----|----|----|----|
|state|enum of the state of the container|false|enum (INIT, STARTED, READY)||
+### KerberosPrincipal
+
+The kerberos principal info of the user who launches the service.
+
+|Name|Description|Required|Schema|Default|
+|----|----|----|----|----|
+|principal_name|The principal name of the user who launches the service.|false|string||
+|keytab|The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.|false|string||
+
### PlacementPolicy
@@ -342,7 +351,7 @@ a service resource has the following attributes.
|state|State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.|false|ServiceState||
|quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object||
|queue|The YARN queue that this service should be submitted to.|false|string||
-
+|kerberos_principal | The principal info of the user who launches the service|false||
### ServiceState
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org