You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2016/04/19 20:27:41 UTC

ambari git commit: AMBARI-15961. Kerberos wizard stuck trying to schedule service check operation (rlevas)

Repository: ambari
Updated Branches:
  refs/heads/branch-2.2.2 b611d806c -> 633cc659d


AMBARI-15961. Kerberos wizard stuck trying to schedule service check operation (rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/633cc659
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/633cc659
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/633cc659

Branch: refs/heads/branch-2.2.2
Commit: 633cc659d52e412b4d294f6bf34ca6e52075f940
Parents: b611d80
Author: Robert Levas <rl...@hortonworks.com>
Authored: Tue Apr 19 14:27:27 2016 -0400
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Tue Apr 19 14:27:34 2016 -0400

----------------------------------------------------------------------
 .../server/controller/KerberosHelperImpl.java   | 308 ++++++++-----------
 .../kerberos/ADKerberosOperationHandler.java    |   9 +-
 .../kerberos/CreatePrincipalsServerAction.java  | 149 +++++----
 .../kerberos/DestroyPrincipalsServerAction.java |  60 ++--
 ...erberosMissingAdminCredentialsException.java |  32 +-
 .../kerberos/KerberosOperationHandler.java      |   2 +
 ...KerberosPrincipalAlreadyExistsException.java |  45 +++
 .../KerberosPrincipalDoesNotExistException.java |  45 +++
 .../kerberos/MITKerberosOperationHandler.java   |  34 +-
 .../AMBARI_METRICS/0.1.0/kerberos.json          |   3 +
 .../server/controller/KerberosHelperTest.java   |  67 +---
 11 files changed, 415 insertions(+), 339 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 737b0aa..dc4829d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -1235,6 +1235,7 @@ public class KerberosHelperImpl implements KerberosHelper {
                 false,
                 kerberosEnvProperties,
                 kerberosOperationHandler,
+                false,
                 null);
 
             if (result == null) {
@@ -1288,16 +1289,7 @@ public class KerberosHelperImpl implements KerberosHelper {
     if (kerberosDetails.manageIdentities()) {
       PrincipalKeyCredential credentials = getKDCAdministratorCredentials(cluster.getClusterName());
       if (credentials == null) {
-        throw new KerberosMissingAdminCredentialsException(
-            "Missing KDC administrator credentials.\n" +
-                "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
-                "This may be done by issuing a POST to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
-                "{\n" +
-                "  \"Credential\" : {\n" +
-                "    \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
-                "  }\n" +
-                "}"
-        );
+        throw new KerberosMissingAdminCredentialsException();
       } else {
         KerberosOperationHandler operationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
 
@@ -1346,16 +1338,7 @@ public class KerberosHelperImpl implements KerberosHelper {
 
           // need to throw this outside of the try/catch so it isn't caught
           if (missingCredentials) {
-            throw new KerberosMissingAdminCredentialsException(
-                "Invalid KDC administrator credentials.\n" +
-                    "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
-                    "This may be done by issuing a POST to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
-                    "{\n" +
-                    "  \"Credential\" : {\n" +
-                    "    \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
-                    "  }\n" +
-                    "}"
-            );
+            throw new KerberosMissingAdminCredentialsException();
           }
         }
       }
@@ -1548,188 +1531,138 @@ public class KerberosHelperImpl implements KerberosHelper {
         throw new AmbariException("The properties map must not be null.  It is needed to store data related to the service check identity");
       }
 
-      Map<String, Service> services = cluster.getServices();
+      List<ServiceComponentHost> serviceComponentHostsToProcess = new ArrayList<ServiceComponentHost>();
+      KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster);
+      KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
 
-      if ((services != null) && !services.isEmpty()) {
-        String clusterName = cluster.getClusterName();
-        Map<String, Host> hosts = clusters.getHostsForCluster(clusterName);
-
-        if ((hosts != null) && !hosts.isEmpty()) {
-          List<ServiceComponentHost> serviceComponentHostsToProcess = new ArrayList<ServiceComponentHost>();
-          KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster);
-          KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
-          Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
-
-          // This is needed to help determine which hosts to perform actions for and create tasks for.
-          Set<String> hostsWithValidKerberosClient = getHostsWithValidKerberosClient(cluster);
-
-          // Create a temporary directory to store metadata needed to complete this task.  Information
-          // such as which principals and keytabs files to create as well as what configurations need
-          // to be update are stored in data files in this directory. Any keytab files are stored in
-          // this directory until they are distributed to their appropriate hosts.
-          File dataDirectory = createTemporaryDirectory();
-
-          // Create the file used to store details about principals and keytabs to create
-          File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
-
-          // Create a special identity for the test user
-          KerberosIdentityDescriptor identity = new KerberosIdentityDescriptor(new HashMap<String, Object>() {
-            {
-              put("principal",
-                  new HashMap<String, Object>() {
-                    {
-                      put("value", "${kerberos-env/service_check_principal_name}@${realm}");
-                      put("type", "user");
-                    }
-                  });
-              put("keytab",
-                  new HashMap<String, Object>() {
-                    {
-                      put("file", "${keytab_dir}/kerberos.service_check.${short_date}.keytab");
-
-                      put("owner", new HashMap<String, Object>() {{
-                        put("name", "${cluster-env/smokeuser}");
-                        put("access", "rw");
-                      }});
-
-                      put("group", new HashMap<String, Object>() {{
-                        put("name", "${cluster-env/user_group}");
-                        put("access", "r");
-                      }});
-
-                      put("cachable", "false");
-                    }
-                  });
-            }
-          });
+      Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
 
-          try {
-            // Iterate over the hosts in the cluster to find the components installed in each.  For each
-            // component (aka service component host - sch) determine the configuration updates and
-            // and the principals an keytabs to create.
-            for (Host host : hosts.values()) {
-              String hostname = host.getHostName();
-
-              // Get a list of components on the current host
-              List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts(hostname);
-
-              if ((serviceComponentHosts != null) && !serviceComponentHosts.isEmpty()) {
-                // Calculate the current host-specific configurations. These will be used to replace
-                // variables within the Kerberos descriptor data
-                Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, hostname, kerberosDescriptorProperties);
-
-                // Add a short date value
-                configurations.get("").put("short_date", new SimpleDateFormat("MMddyy").format(new Date()));
-
-                // Iterate over the components installed on the current host to get the service and
-                // component-level Kerberos descriptors in order to determine which principals,
-                // keytab files, and configurations need to be created or updated.
-                for (ServiceComponentHost sch : serviceComponentHosts) {
-                  String serviceName = sch.getServiceName();
-                  String componentName = sch.getServiceComponentName();
-
-                  // If the current ServiceComponentHost represents the KERBEROS/KERBEROS_CLIENT and
-                  // indicates that the KERBEROS_CLIENT component is in the INSTALLED state, add the
-                  // current host to the set of hosts that should be handled...
-                  if (Service.Type.KERBEROS.name().equals(serviceName) &&
-                      Role.KERBEROS_CLIENT.name().equals(componentName) &&
-                      (sch.getState() == State.INSTALLED)) {
-                    hostsWithValidKerberosClient.add(hostname);
-
-                    int identitiesAdded = 0;
-
-                    // Lazily create the KerberosIdentityDataFileWriter instance...
-                    if (kerberosIdentityDataFileWriter == null) {
-                      kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
-                    }
+      // This is needed to help determine which hosts to perform actions for and create tasks for.
+      Set<String> hostsWithValidKerberosClient = getHostsWithValidKerberosClient(cluster);
 
-                    // Add service-level principals (and keytabs)
-                    identitiesAdded += addIdentities(kerberosIdentityDataFileWriter, Collections.singleton(identity),
-                        null, hostname, serviceName, componentName, null, configurations);
+      // Create a temporary directory to store metadata needed to complete this task.  Information
+      // such as which principals and keytabs files to create as well as what configurations need
+      // to be update are stored in data files in this directory. Any keytab files are stored in
+      // this directory until they are distributed to their appropriate hosts.
+      File dataDirectory = createTemporaryDirectory();
 
-                    if (identitiesAdded > 0) {
-                      // Add the relevant principal name and keytab file data to the command params state
-                      if (!commandParameters.containsKey("principal_name") || !commandParameters.containsKey("keytab_file")) {
-                        commandParameters.put("principal_name",
-                            variableReplacementHelper.replaceVariables(identity.getPrincipalDescriptor().getValue(), configurations));
-                        commandParameters.put("keytab_file",
-                            variableReplacementHelper.replaceVariables(identity.getKeytabDescriptor().getFile(), configurations));
-                      }
+      // Create the file used to store details about principals and keytabs to create
+      File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
 
-                      serviceComponentHostsToProcess.add(sch);
-                    }
-                  }
-                }
-              }
-            }
-          } catch (IOException e) {
-            String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
-            LOG.error(message);
-            throw new AmbariException(message, e);
-          } finally {
-            if (kerberosIdentityDataFileWriter != null) {
-              // Make sure the data file is closed
-              try {
-                kerberosIdentityDataFileWriter.close();
-              } catch (IOException e) {
-                LOG.warn("Failed to close the index file writer", e);
-              }
-            }
-          }
+      // Calculate the current non-host-specific configurations. These will be used to replace
+      // variables within the Kerberos descriptor data
+      Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, null, kerberosDescriptorProperties);
 
-          // If there are ServiceComponentHosts to process, make sure the administrator credential
-          // are available
-          if (!serviceComponentHostsToProcess.isEmpty()) {
-            try {
-              validateKDCCredentials(kerberosDetails, cluster);
-            } catch (KerberosOperationException e) {
-              try {
-                FileUtils.deleteDirectory(dataDirectory);
-              } catch (Throwable t) {
-                LOG.warn(String.format("The data directory (%s) was not deleted due to an error condition - {%s}",
-                    dataDirectory.getAbsolutePath(), t.getMessage()), t);
-              }
+      String principal = variableReplacementHelper.replaceVariables("${kerberos-env/service_check_principal_name}@${realm}", configurations);
+      String principalType = "user";
 
-              throw e;
+      String keytabFilePath = variableReplacementHelper.replaceVariables("${keytab_dir}/kerberos.service_check.${short_date}.keytab", configurations);
+      String keytabFileOwnerName = variableReplacementHelper.replaceVariables("${cluster-env/smokeuser}", configurations);
+      String keytabFileOwnerAccess = "rw";
+      String keytabFileGroupName = variableReplacementHelper.replaceVariables("${cluster-env/user_group}", configurations);
+      String keytabFileGroupAccess = "r";
+
+      // Add the relevant principal name and keytab file data to the command params state
+      commandParameters.put("principal_name", principal);
+      commandParameters.put("keytab_file", keytabFilePath);
+
+      try {
+        List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name());
+
+        if ((serviceComponentHosts != null) && !serviceComponentHosts.isEmpty()) {
+          kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
+
+          // Iterate over the KERBEROS_CLIENT service component hosts to get the service and
+          // component-level Kerberos descriptors in order to determine which principals,
+          // keytab files needed to be created or updated.
+          for (ServiceComponentHost sch : serviceComponentHosts) {
+            if (sch.getState() == State.INSTALLED) {
+              String hostname = sch.getHostName();
+
+              kerberosIdentityDataFileWriter.writeRecord(
+                  hostname,
+                  Service.Type.KERBEROS.name(),
+                  Role.KERBEROS_CLIENT.name(),
+                  principal,
+                  principalType,
+                  keytabFilePath,
+                  keytabFileOwnerName,
+                  keytabFileOwnerAccess,
+                  keytabFileGroupName,
+                  keytabFileGroupAccess,
+                  "false");
+
+              hostsWithValidKerberosClient.add(hostname);
+              serviceComponentHostsToProcess.add(sch);
             }
           }
+        }
+      } catch (IOException e) {
+        String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
+        LOG.error(message);
+        throw new AmbariException(message, e);
+      } finally {
+        if (kerberosIdentityDataFileWriter != null) {
+          // Make sure the data file is closed
+          try {
+            kerberosIdentityDataFileWriter.close();
+          } catch (IOException e) {
+            LOG.warn("Failed to close the index file writer", e);
+          }
+        }
+      }
 
-          // Always set up the necessary stages to perform the tasks needed to complete the operation.
-          // Some stages may be no-ops, this is expected.
-          // Gather data needed to create stages and tasks...
-          Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
-          String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
-          Map<String, String> hostParams = customCommandExecutionHelper.createDefaultHostParams(cluster);
-          String hostParamsJson = StageUtils.getGson().toJson(hostParams);
-          String ambariServerHostname = StageUtils.getHostName();
-          ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(
-              "AMBARI_SERVER",
-              ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
-              System.currentTimeMillis());
-          RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
-
-          // If a RequestStageContainer does not already exist, create a new one...
-          if (requestStageContainer == null) {
-            requestStageContainer = new RequestStageContainer(
-                actionManager.getNextRequestId(),
-                null,
-                requestFactory,
-                actionManager);
+      // If there are ServiceComponentHosts to process, make sure the administrator credential
+      // are available
+      if (!serviceComponentHostsToProcess.isEmpty()) {
+        try {
+          validateKDCCredentials(kerberosDetails, cluster);
+        } catch (KerberosOperationException e) {
+          try {
+            FileUtils.deleteDirectory(dataDirectory);
+          } catch (Throwable t) {
+            LOG.warn(String.format("The data directory (%s) was not deleted due to an error condition - {%s}",
+                dataDirectory.getAbsolutePath(), t.getMessage()), t);
           }
 
-          // Use the handler implementation to setup the relevant stages.
-          // Set the service/component filter to an empty map since the service/component processing
-          // was done above.
-          handler.createStages(cluster,
-              clusterHostInfoJson, hostParamsJson, event, roleCommandOrder, kerberosDetails,
-              dataDirectory, requestStageContainer, serviceComponentHostsToProcess,
-              Collections.<String, Collection<String>>emptyMap(), null, null, hostsWithValidKerberosClient);
+          throw e;
+        }
+      }
 
+      // Always set up the necessary stages to perform the tasks needed to complete the operation.
+      // Some stages may be no-ops, this is expected.
+      // Gather data needed to create stages and tasks...
+      Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
+      String clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
+      Map<String, String> hostParams = customCommandExecutionHelper.createDefaultHostParams(cluster);
+      String hostParamsJson = StageUtils.getGson().toJson(hostParams);
+      String ambariServerHostname = StageUtils.getHostName();
+      ServiceComponentHostServerActionEvent event = new ServiceComponentHostServerActionEvent(
+          "AMBARI_SERVER",
+          ambariServerHostname, // TODO: Choose a random hostname from the cluster. All tasks for the AMBARI_SERVER service will be executed on this Ambari server
+          System.currentTimeMillis());
+      RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
 
-          handler.addFinalizeOperationStage(cluster, clusterHostInfoJson, hostParamsJson, event,
-              dataDirectory, roleCommandOrder, requestStageContainer);
-        }
+      // If a RequestStageContainer does not already exist, create a new one...
+      if (requestStageContainer == null) {
+        requestStageContainer = new RequestStageContainer(
+            actionManager.getNextRequestId(),
+            null,
+            requestFactory,
+            actionManager);
       }
+
+      // Use the handler implementation to setup the relevant stages.
+      // Set the service/component filter to an empty map since the service/component processing
+      // was done above.
+      handler.createStages(cluster,
+          clusterHostInfoJson, hostParamsJson, event, roleCommandOrder, kerberosDetails,
+          dataDirectory, requestStageContainer, serviceComponentHostsToProcess,
+          Collections.<String, Collection<String>>emptyMap(), null, null, hostsWithValidKerberosClient);
+
+
+      handler.addFinalizeOperationStage(cluster, clusterHostInfoJson, hostParamsJson, event,
+          dataDirectory, roleCommandOrder, requestStageContainer);
     }
 
     return requestStageContainer;
@@ -2277,6 +2210,7 @@ public class KerberosHelperImpl implements KerberosHelper {
 
     // Add the current cluster's name
     generalProperties.put("cluster_name", cluster.getClusterName());
+    generalProperties.put("short_date", new SimpleDateFormat("MMddyy").format(new Date()));
 
     // add clusterHostInfo config
     if (configurations.get("clusterHostInfo") == null) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java
index a1e1544..c236a27 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ADKerberosOperationHandler.java
@@ -235,6 +235,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
    * @param password  a String containing the password to use when creating the principal
    * @param service   a boolean value indicating whether the principal is to be created as a service principal or not
    * @return an Integer declaring the generated key number
+   * @throws KerberosPrincipalAlreadyExistsException if the principal already exists
    * @throws KerberosOperationException
    */
   @Override
@@ -243,13 +244,15 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
     if (!isOpen()) {
       throw new KerberosOperationException("This operation handler has not been opened");
     }
-
     if (principal == null) {
       throw new KerberosOperationException("principal is null");
     }
     if (password == null) {
       throw new KerberosOperationException("principal password is null");
     }
+    if (principalExists(principal)) {
+      throw new KerberosPrincipalAlreadyExistsException(principal);
+    }
 
     DeconstructedPrincipal deconstructedPrincipal = createDeconstructPrincipal(principal);
 
@@ -333,6 +336,7 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
    * @param principal a String containing the principal to update
    * @param password  a String containing the password to set
    * @return an Integer declaring the new key number
+   * @throws KerberosPrincipalDoesNotExistException if the principal does not exist
    * @throws KerberosOperationException
    */
   @Override
@@ -346,6 +350,9 @@ public class ADKerberosOperationHandler extends KerberosOperationHandler {
     if (password == null) {
       throw new KerberosOperationException("principal password is null");
     }
+    if(!principalExists(principal)) {
+      throw new KerberosPrincipalDoesNotExistException(principal);
+    }
 
     DeconstructedPrincipal deconstructPrincipal = createDeconstructPrincipal(principal);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
index 55e046e..99487c9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
@@ -31,7 +31,9 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -64,6 +66,12 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
   private SecurePasswordHelper securePasswordHelper;
 
   /**
+   * A set of visited principal names used to prevent unnecessary processing on already processed
+   * principal names
+   */
+  private Set<String> seenPrincipals = new HashSet<String>();
+
+  /**
    * Called to execute this action.  Upon invocation, calls
    * {@link org.apache.ambari.server.serveraction.kerberos.KerberosServerAction#processIdentities(java.util.Map)}
    * to iterate through the Kerberos identity metadata and call
@@ -113,43 +121,49 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       throws AmbariException {
     CommandReport commandReport = null;
 
-    boolean processPrincipal;
-    boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL));
+    //  Only process this principal name if we haven't already processed it
+    if (!seenPrincipals.contains(evaluatedPrincipal)) {
+      seenPrincipals.add(evaluatedPrincipal);
 
-    if (regenerateKeytabs) {
-      processPrincipal = true;
-    } else {
-      KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+      boolean processPrincipal;
+      boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL));
 
-      if (kerberosPrincipalEntity == null) {
-        // This principal has not been processed before, process it.
+      if (regenerateKeytabs) {
         processPrincipal = true;
-      } else if (!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) {
-        // This principal has been processed and a keytab file has been cached for it... do not process it.
-        processPrincipal = false;
-      } else if (kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) {
-        // This principal has been processed and a keytab file has been distributed... do not process it.
-        processPrincipal = false;
       } else {
-        // This principal has been processed but a keytab file for it has been distributed... process it.
-        processPrincipal = true;
+        KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+
+        if (kerberosPrincipalEntity == null) {
+          // This principal has not been processed before, process it.
+          processPrincipal = true;
+        } else if (!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) {
+          // This principal has been processed and a keytab file has been cached for it... do not process it.
+          processPrincipal = false;
+        } else if (kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) {
+          // This principal has been processed and a keytab file has been distributed... do not process it.
+          processPrincipal = false;
+        } else {
+          // This principal has been processed but a keytab file for it has been distributed... process it.
+          processPrincipal = true;
+        }
       }
-    }
       Map<String, String> principalPasswordMap = getPrincipalPasswordMap(requestSharedDataContext);
       Map<String, Integer> principalKeyNumberMap = getPrincipalKeyNumberMap(requestSharedDataContext);
 
-    if (processPrincipal) {
-      String password = principalPasswordMap.get(evaluatedPrincipal);
+      if (processPrincipal) {
+        String password = principalPasswordMap.get(evaluatedPrincipal);
 
-      if (password == null) {
-        boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
-        CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, actionLog);
+        if (password == null) {
+          boolean servicePrincipal = "service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
 
-        if (result == null) {
-          commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
-        } else {
-          principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
-          principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber());
+          CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, servicePrincipal, kerberosConfiguration, operationHandler, regenerateKeytabs, actionLog);
+
+          if (result == null) {
+            commandReport = createCommandReport(1, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+          } else {
+            principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
+            principalKeyNumberMap.put(evaluatedPrincipal, result.getKeyNumber());
+          }
         }
       }
     }
@@ -165,16 +179,16 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
    *                                 principal is a user principal
    * @param kerberosConfiguration    the kerberos-env configuration properties
    * @param kerberosOperationHandler the KerberosOperationHandler for the relevant KDC
-   * @param actionLog                the logger (may be null if no logging is desired)
-   * @return a CreatePrincipalResult containing the generated password and key number value
+   * @param regenerateKeytabs        true if this was triggered in response to regenerating keytab files; false otherwize
+   * @param actionLog                the logger (may be null if no logging is desired)  @return a CreatePrincipalResult containing the generated password and key number value
    */
   public CreatePrincipalResult createPrincipal(String principal, boolean isServicePrincipal,
                                                Map<String, String> kerberosConfiguration,
                                                KerberosOperationHandler kerberosOperationHandler,
-                                               ActionLog actionLog) {
+                                               boolean regenerateKeytabs, ActionLog actionLog) {
     CreatePrincipalResult result = null;
 
-    String message = String.format("Creating principal, %s", principal);
+    String message = String.format("Processing principal, %s", principal);
     LOG.info(message);
     if (actionLog != null) {
       actionLog.writeStdOut(message);
@@ -203,48 +217,61 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       minWhitespace = toInt(kerberosConfiguration.get("password_min_whitespace"));
     }
 
+    // Create a new password since we need to know what it is.
     String password = securePasswordHelper.createSecurePassword(length, minLowercaseLetters, minUppercaseLetters, minDigits, minPunctuation, minWhitespace);
 
     try {
+      boolean created;
+      Integer keyNumber;
+
+      if (regenerateKeytabs) {
+        try {
+          keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
+          created = false;
+        } catch (KerberosPrincipalDoesNotExistException e) {
+          message = String.format("Principal, %s, does not exist, creating new principal", principal);
+          LOG.warn(message);
+          if (actionLog != null) {
+            actionLog.writeStdOut(message);
+          }
 
-      if (kerberosOperationHandler.principalExists(principal)) {
-        // Create a new password since we need to know what it is.
-        // A new password/key would have been generated after exporting the keytab anyways.
-        message = String.format("Principal, %s, already exists, setting new password", principal);
-        LOG.warn(message);
-        if (actionLog != null) {
-          actionLog.writeStdOut(message);
+          keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+          created = true;
         }
-
-        Integer keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
-
-        if (keyNumber != null) {
-          result = new CreatePrincipalResult(principal, password, keyNumber);
-          message = String.format("Successfully set password for %s", principal);
-          LOG.debug(message);
-        } else {
-          message = String.format("Failed to set password for %s - unknown reason", principal);
-          LOG.error(message);
+      } else {
+        try {
+          keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+          created = true;
+        } catch (KerberosPrincipalAlreadyExistsException e) {
+          message = String.format("Principal, %s, already exists, setting new password", principal);
+          LOG.warn(message);
           if (actionLog != null) {
-            actionLog.writeStdErr(message);
+            actionLog.writeStdOut(message);
           }
+
+          keyNumber = kerberosOperationHandler.setPrincipalPassword(principal, password);
+          created = false;
         }
-      } else {
-        message = String.format("Creating new principal, %s", principal);
-        LOG.debug(message);
+      }
 
-        Integer keyNumber = kerberosOperationHandler.createPrincipal(principal, password, isServicePrincipal);
+      if (keyNumber != null) {
+        result = new CreatePrincipalResult(principal, password, keyNumber);
 
-        if (keyNumber != null) {
-          result = new CreatePrincipalResult(principal, password, keyNumber);
+        if (created) {
           message = String.format("Successfully created new principal, %s", principal);
-          LOG.debug(message);
         } else {
+          message = String.format("Successfully set password for %s", principal);
+        }
+        LOG.debug(message);
+      } else {
+        if (created) {
           message = String.format("Failed to create principal, %s - unknown reason", principal);
-          LOG.error(message);
-          if (actionLog != null) {
-            actionLog.writeStdErr(message);
-          }
+        } else {
+          message = String.format("Failed to set password for %s - unknown reason", principal);
+        }
+        LOG.error(message);
+        if (actionLog != null) {
+          actionLog.writeStdErr(message);
         }
       }
 
@@ -253,7 +280,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction {
       }
 
     } catch (KerberosOperationException e) {
-      message = String.format("Failed to create principal, %s - %s", principal, e.getMessage());
+      message = String.format("Failed to create or update principal, %s - %s", principal, e.getMessage());
       LOG.error(message, e);
       if (actionLog != null) {
         actionLog.writeStdErr(message);

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
index 93daae8..cc635c4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/DestroyPrincipalsServerAction.java
@@ -27,7 +27,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -46,6 +48,12 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
   private KerberosPrincipalDAO kerberosPrincipalDAO;
 
   /**
+   * A set of visited principal names used to prevent unnecessary processing on already processed
+   * principal names
+   */
+  private Set<String> seenPrincipals = new HashSet<String>();
+
+  /**
    * Called to execute this action.  Upon invocation, calls
    * {@link KerberosServerAction#processIdentities(java.util.Map)}
    * to iterate through the Kerberos identity metadata and call
@@ -86,39 +94,43 @@ public class DestroyPrincipalsServerAction extends KerberosServerAction {
                                           Map<String, Object> requestSharedDataContext)
       throws AmbariException {
 
-    String message = String.format("Destroying identity, %s", evaluatedPrincipal);
-    LOG.info(message);
-    actionLog.writeStdOut(message);
+    // Only process this principal if we haven't already processed it
+    if(!seenPrincipals.contains(evaluatedPrincipal)) {
+      seenPrincipals.add(evaluatedPrincipal);
 
-    try {
-      operationHandler.removePrincipal(evaluatedPrincipal);
-    } catch (KerberosOperationException e) {
-      message = String.format("Failed to remove identity for %s from the KDC - %s", evaluatedPrincipal, e.getMessage());
-      LOG.warn(message);
-      actionLog.writeStdErr(message);
-    }
+      String message = String.format("Destroying identity, %s", evaluatedPrincipal);
+      LOG.info(message);
+      actionLog.writeStdOut(message);
 
-    try {
-      KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
+      try {
+        operationHandler.removePrincipal(evaluatedPrincipal);
+      } catch (KerberosOperationException e) {
+        message = String.format("Failed to remove identity for %s from the KDC - %s", evaluatedPrincipal, e.getMessage());
+        LOG.warn(message);
+        actionLog.writeStdErr(message);
+      }
+
+      try {
+        KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal);
 
-      if(principalEntity != null) {
-        String cachedKeytabPath = principalEntity.getCachedKeytabPath();
+        if (principalEntity != null) {
+          String cachedKeytabPath = principalEntity.getCachedKeytabPath();
 
-        kerberosPrincipalDAO.remove(principalEntity);
+          kerberosPrincipalDAO.remove(principalEntity);
 
-        // If a cached  keytabs file exists for this principal, delete it.
-        if (cachedKeytabPath != null) {
-          if (!new File(cachedKeytabPath).delete()) {
-            LOG.debug(String.format("Failed to remove cached keytab for %s", evaluatedPrincipal));
+          // If a cached  keytabs file exists for this principal, delete it.
+          if (cachedKeytabPath != null) {
+            if (!new File(cachedKeytabPath).delete()) {
+              LOG.debug(String.format("Failed to remove cached keytab for %s", evaluatedPrincipal));
+            }
           }
         }
+      } catch (Throwable t) {
+        message = String.format("Failed to remove identity for %s from the Ambari database - %s", evaluatedPrincipal, t.getMessage());
+        LOG.warn(message);
+        actionLog.writeStdErr(message);
       }
     }
-    catch (Throwable t) {
-      message = String.format("Failed to remove identity for %s from the Ambari database - %s", evaluatedPrincipal, t.getMessage());
-      LOG.warn(message);
-      actionLog.writeStdErr(message);
-    }
 
     // There is no reason to fail this task if an identity was not removed. The cluster will work
     // just fine if this cleanup process fails.

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosMissingAdminCredentialsException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosMissingAdminCredentialsException.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosMissingAdminCredentialsException.java
index ddb2769..323af59 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosMissingAdminCredentialsException.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosMissingAdminCredentialsException.java
@@ -23,6 +23,25 @@ package org.apache.ambari.server.serveraction.kerberos;
  */
 public class KerberosMissingAdminCredentialsException extends KerberosOperationException {
   /**
+   * The default error message to use when handling this exception
+   */
+  private static final String DEFAULT_MESSAGE = "Missing KDC administrator credentials.\n" +
+      "The KDC administrator credentials must be set as a persisted or temporary credential resource." +
+      "This may be done by issuing a POST to the /api/v1/clusters/:clusterName/credentials/kdc.admin.credential API entry point with the following payload:\n" +
+      "{\n" +
+      "  \"Credential\" : {\n" +
+      "    \"principal\" : \"(PRINCIPAL)\", \"key\" : \"(PASSWORD)\", \"type\" : \"(persisted|temporary)\"}\n" +
+      "  }\n" +
+      "}";
+
+  /**
+   * Constructor using the default missing credentials message.
+   */
+  public KerberosMissingAdminCredentialsException() {
+    this(DEFAULT_MESSAGE);
+  }
+
+  /**
    * Constructor.
    *
    * @param message error message
@@ -32,10 +51,19 @@ public class KerberosMissingAdminCredentialsException extends KerberosOperationE
   }
 
   /**
+   * Constructor using the default message.
+   *
+   * @param cause   root cause
+   */
+  public KerberosMissingAdminCredentialsException(Throwable cause) {
+    this(DEFAULT_MESSAGE, cause);
+  }
+
+  /**
    * Constructor.
    *
-   * @param message  error message
-   * @param cause    root cause
+   * @param message error message
+   * @param cause   root cause
    */
   public KerberosMissingAdminCredentialsException(String message, Throwable cause) {
     super(message, cause);

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosOperationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosOperationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosOperationHandler.java
index e3d31de..1a50ce5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosOperationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosOperationHandler.java
@@ -246,6 +246,7 @@ public abstract class KerberosOperationHandler {
    * @param service   a boolean value indicating whether the principal is to be created as a service principal or not
    * @return an Integer declaring the generated key number
    * @throws KerberosOperationException
+   * @throws KerberosPrincipalAlreadyExistsException if the principal already exists
    */
   public abstract Integer createPrincipal(String principal, String password, boolean service)
       throws KerberosOperationException;
@@ -259,6 +260,7 @@ public abstract class KerberosOperationHandler {
    * @param password  a String containing the password to set
    * @return an Integer declaring the new key number
    * @throws KerberosOperationException
+   * @throws KerberosPrincipalDoesNotExistException if the principal does not exist
    */
   public abstract Integer setPrincipalPassword(String principal, String password)
       throws KerberosOperationException;

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalAlreadyExistsException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalAlreadyExistsException.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalAlreadyExistsException.java
new file mode 100644
index 0000000..3419c0b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalAlreadyExistsException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.kerberos;
+
+/**
+ * KerberosPrincipalAlreadyExistsException is a KerberosOperationException thrown in the event a
+ * request to create a new princials was made but the princial already exists in the KDC.
+ */
+public class KerberosPrincipalAlreadyExistsException extends KerberosOperationException {
+
+  /**
+   * Creates a new KerberosPrincipalAlreadyExistsException with a message
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   */
+  public KerberosPrincipalAlreadyExistsException(String message) {
+    super(message);
+  }
+
+  /**
+   * Creates a new KerberosPrincipalAlreadyExistsException with a message and a cause
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   * @param cause   a Throwable declaring the previously thrown Throwable that led to this exception
+   */
+  public KerberosPrincipalAlreadyExistsException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalDoesNotExistException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalDoesNotExistException.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalDoesNotExistException.java
new file mode 100644
index 0000000..5618547
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosPrincipalDoesNotExistException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.kerberos;
+
+/**
+ * KerberosPrincipalDoesNotExistException is a KerberosOperationException thrown in the event a
+ * request to modify an existing princial was made but the princial does not exist in the KDC.
+ */
+public class KerberosPrincipalDoesNotExistException extends KerberosOperationException {
+
+  /**
+   * Creates a new KerberosPrincipalDoesNotExistException with a message
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   */
+  public KerberosPrincipalDoesNotExistException(String message) {
+    super(message);
+  }
+
+  /**
+   * Creates a new KerberosPrincipalDoesNotExistException with a message and a cause
+   *
+   * @param message a String containing the message indicating the reason for this exception
+   * @param cause   a Throwable declaring the previously thrown Throwable that led to this exception
+   */
+  public KerberosPrincipalDoesNotExistException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
index 90858fb..f48052f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
@@ -165,10 +165,11 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
    * @param password  a String containing the password to use when creating the principal
    * @param service   a boolean value indicating whether the principal is to be created as a service principal or not
    * @return an Integer declaring the generated key number
-   * @throws KerberosKDCConnectionException       if a connection to the KDC cannot be made
-   * @throws KerberosAdminAuthenticationException if the administrator credentials fail to authenticate
-   * @throws KerberosRealmException               if the realm does not map to a KDC
-   * @throws KerberosOperationException           if an unexpected error occurred
+   * @throws KerberosKDCConnectionException          if a connection to the KDC cannot be made
+   * @throws KerberosAdminAuthenticationException    if the administrator credentials fail to authenticate
+   * @throws KerberosRealmException                  if the realm does not map to a KDC
+   * @throws KerberosPrincipalAlreadyExistsException if the principal already exists
+   * @throws KerberosOperationException              if an unexpected error occurred
    */
   @Override
   public Integer createPrincipal(String principal, String password, boolean service)
@@ -193,6 +194,8 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
       String stdOut = result.getStdout();
       if ((stdOut != null) && stdOut.contains(String.format("Principal \"%s\" created", principal))) {
         return getKeyNumber(principal);
+      } else if ((stdOut != null) && stdOut.contains(String.format("Principal or policy already exists while creating \"%s\"", principal))) {
+        throw new KerberosPrincipalAlreadyExistsException(principal);
       } else {
         LOG.error("Failed to execute kadmin query: add_principal -pw \"********\" {} {}\nSTDOUT: {}\nSTDERR: {}",
             (createAttributes == null) ? "" : createAttributes, principal, stdOut, result.getStderr());
@@ -211,10 +214,11 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
    * @param principal a String containing the principal to update
    * @param password  a String containing the password to set
    * @return an Integer declaring the new key number
-   * @throws KerberosKDCConnectionException       if a connection to the KDC cannot be made
-   * @throws KerberosAdminAuthenticationException if the administrator credentials fail to authenticate
-   * @throws KerberosRealmException               if the realm does not map to a KDC
-   * @throws KerberosOperationException           if an unexpected error occurred
+   * @throws KerberosKDCConnectionException         if a connection to the KDC cannot be made
+   * @throws KerberosAdminAuthenticationException   if the administrator credentials fail to authenticate
+   * @throws KerberosRealmException                 if the realm does not map to a KDC
+   * @throws KerberosPrincipalDoesNotExistException if the principal does not exist
+   * @throws KerberosOperationException             if an unexpected error occurred
    */
   @Override
   public Integer setPrincipalPassword(String principal, String password) throws KerberosOperationException {
@@ -228,9 +232,19 @@ public class MITKerberosOperationHandler extends KerberosOperationHandler {
       throw new KerberosOperationException("Failed to set password - no password specified");
     } else {
       // Create the kdamin query:  change_password <-randkey|-pw <password>> <principal>
-      invokeKAdmin(String.format("change_password -pw \"%s\" %s", password, principal));
+      ShellCommandUtil.Result result = invokeKAdmin(String.format("change_password -pw \"%s\" %s", password, principal));
 
-      return getKeyNumber(principal);
+      String stdOut = result.getStdout();
+      if ((stdOut != null) && stdOut.contains(String.format("Password for \"%s\" changed", principal))) {
+        return getKeyNumber(principal);
+      } else if ((stdOut != null) && stdOut.contains("Principal does not exist")) {
+        throw new KerberosPrincipalDoesNotExistException(principal);
+      } else {
+        LOG.error("Failed to execute kadmin query: change_password -pw \"********\" {} \nSTDOUT: {}\nSTDERR: {}",
+            principal, stdOut, result.getStderr());
+        throw new KerberosOperationException(String.format("Failed to update password for %s\nSTDOUT: %s\nSTDERR: %s",
+            principal, stdOut, result.getStderr()));
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/kerberos.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/kerberos.json
index 34de6a8..d84ed02 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/kerberos.json
@@ -12,6 +12,9 @@
           "name": "METRICS_COLLECTOR",
           "identities": [
             {
+              "name": "/HDFS/NAMENODE/hdfs"
+            },
+            {
               "name": "ams_hbase_master_hbase",
               "principal": {
                 "value": "amshbase/_HOST@${realm}",

http://git-wip-us.apache.org/repos/asf/ambari/blob/633cc659/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
index eefaf04..feb11a1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -2594,7 +2594,7 @@ public class KerberosHelperTest extends EasyMockSupport {
     Capture<? extends String> capturePrincipalForKeytab = newCapture(CaptureType.ALL);
 
     CreatePrincipalsServerAction createPrincipalsServerAction = injector.getInstance(CreatePrincipalsServerAction.class);
-    expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), anyObject(Map.class),  anyObject(KerberosOperationHandler.class), isNull(ActionLog.class)))
+    expect(createPrincipalsServerAction.createPrincipal(capture(capturePrincipal), eq(false), anyObject(Map.class),  anyObject(KerberosOperationHandler.class), eq(false), isNull(ActionLog.class)))
         .andReturn(new CreatePrincipalsServerAction.CreatePrincipalResult("anything", "password", 1))
         .times(3);
 
@@ -3205,20 +3205,19 @@ public class KerberosHelperTest extends EasyMockSupport {
     KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
     boolean managingIdentities = !Boolean.FALSE.equals(manageIdentities);
 
-    final Map<String, String> kerberosEnvProperties = createMock(Map.class);
-    expect(kerberosEnvProperties.get("kdc_type")).andReturn("mit-kdc").anyTimes();
-    expect(kerberosEnvProperties.get("realm")).andReturn("FOOBAR.COM").anyTimes();
-
-    expect(kerberosEnvProperties.get("manage_identities"))
-        .andReturn((manageIdentities == null)
+    final Map<String, String> kerberosEnvProperties = new HashMap<String,String>();
+    kerberosEnvProperties.put("kdc_type","mit-kdc");
+    kerberosEnvProperties.put("realm","FOOBAR.COM");
+    kerberosEnvProperties.put("manage_identities","FOOBAR.COM");
+    kerberosEnvProperties.put("manage_identities",
+        (manageIdentities == null)
             ? null
-            : ((manageIdentities) ? "true" : "false"))
-        .anyTimes();
+            : ((manageIdentities) ? "true" : "false"));
 
     final Config kerberosEnvConfig = createMock(Config.class);
     expect(kerberosEnvConfig.getProperties()).andReturn(kerberosEnvProperties).anyTimes();
 
-    final Map<String, String> krb5ConfProperties = createMock(Map.class);
+    final Map<String, String> krb5ConfProperties = new HashMap<String, String>();
 
     final Config krb5ConfConfig = createMock(Config.class);
     expect(krb5ConfConfig.getProperties()).andReturn(krb5ConfProperties).anyTimes();
@@ -3243,20 +3242,6 @@ public class KerberosHelperTest extends EasyMockSupport {
       expect(schKerberosClient.getHostName()).andReturn("host1").anyTimes();
       expect(schKerberosClient.getState()).andReturn(State.INSTALLED).anyTimes();
 
-      final ServiceComponentHost sch1 = createMock(ServiceComponentHost.class);
-      expect(sch1.getServiceName()).andReturn("SERVICE1").anyTimes();
-      expect(sch1.getServiceComponentName()).andReturn("COMPONENT1").anyTimes();
-      expect(sch1.getHostName()).andReturn("host1").anyTimes();
-
-      final ServiceComponentHost sch2 = createStrictMock(ServiceComponentHost.class);
-      expect(sch2.getServiceName()).andReturn("SERVICE2").anyTimes();
-      expect(sch2.getServiceComponentName()).andReturn("COMPONENT3").anyTimes();
-
-      final ServiceComponentHost sch3 = createStrictMock(ServiceComponentHost.class);
-      expect(sch3.getServiceName()).andReturn("SERVICE3").anyTimes();
-      expect(sch3.getServiceComponentName()).andReturn("COMPONENT3").anyTimes();
-      expect(sch3.getHostName()).andReturn("host1").anyTimes();
-
       final ServiceComponent serviceComponentKerberosClient = createNiceMock(ServiceComponent.class);
       expect(serviceComponentKerberosClient.getName()).andReturn(Role.KERBEROS_CLIENT.name()).anyTimes();
       expect(serviceComponentKerberosClient.getServiceComponentHosts()).andReturn(Collections.singletonMap("host1", schKerberosClient)).anyTimes();
@@ -3290,16 +3275,13 @@ public class KerberosHelperTest extends EasyMockSupport {
             }
           })
           .anyTimes();
-      expect(cluster.getServiceComponentHosts("host1"))
+      expect(cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name()))
           .andReturn(new ArrayList<ServiceComponentHost>() {
             {
-              add(sch1);
-              add(sch2);
-              add(sch3);
               add(schKerberosClient);
             }
           })
-          .once();
+          .anyTimes();
       expect(cluster.getCurrentStackVersion())
           .andReturn(new StackId("HDP", "2.2"))
           .anyTimes();
@@ -3316,21 +3298,11 @@ public class KerberosHelperTest extends EasyMockSupport {
       }).anyTimes();
 
       final Clusters clusters = injector.getInstance(Clusters.class);
-      expect(clusters.getHostsForCluster("c1"))
-          .andReturn(new HashMap<String, Host>() {
-            {
-              put("host1", host);
-            }
-          })
-          .once();
       expect(clusters.getHost("host1"))
           .andReturn(host)
           .once();
 
       final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
-      expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1"))
-          .andReturn(Collections.<String, Map<String, String>>emptyMap())
-          .once();
       expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
           .andReturn(Collections.<String, Map<String, String>>emptyMap())
           .once();
@@ -3496,37 +3468,24 @@ public class KerberosHelperTest extends EasyMockSupport {
           }
         })
         .anyTimes();
-    expect(cluster.getServiceComponentHosts("host1"))
+    expect(cluster.getServiceComponentHosts(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name()))
         .andReturn(new ArrayList<ServiceComponentHost>() {
           {
-            add(sch1);
-            add(sch2);
-            add(sch3);
             add(schKerberosClient);
           }
         })
-        .once();
+        .anyTimes();
     expect(cluster.getCurrentStackVersion())
         .andReturn(new StackId("HDP", "2.2"))
         .anyTimes();
     expect(cluster.getSessionAttributes()).andReturn(new HashMap<String, Object>()).anyTimes();
 
     final Clusters clusters = injector.getInstance(Clusters.class);
-    expect(clusters.getHostsForCluster("c1"))
-        .andReturn(new HashMap<String, Host>() {
-          {
-            put("host1", host);
-          }
-        })
-        .once();
     expect(clusters.getHost("host1"))
         .andReturn(host)
         .once();
 
     final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
-    expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1"))
-        .andReturn(Collections.<String, Map<String, String>>emptyMap())
-        .once();
     expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
         .andReturn(Collections.<String, Map<String, String>>emptyMap())
         .once();