You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2015/01/30 18:47:02 UTC

ambari git commit: AMBARI-9385. Implement Keytab regeneration (rlevas)

Repository: ambari
Updated Branches:
  refs/heads/trunk 4e80ff992 -> a116498e9


AMBARI-9385. Implement Keytab regeneration (rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a116498e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a116498e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a116498e

Branch: refs/heads/trunk
Commit: a116498e92babd241f2bc083bb1ef2325f4ca3d7
Parents: 4e80ff9
Author: Robert Levas <rl...@hortonworks.com>
Authored: Fri Jan 30 10:57:59 2015 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Fri Jan 30 12:46:44 2015 -0500

----------------------------------------------------------------------
 .../resources/ClusterResourceDefinition.java    |  10 +
 .../AmbariManagementControllerImpl.java         |  13 +-
 .../server/controller/KerberosHelper.java       |  96 ++++++-
 .../AmbariManagementControllerImplTest.java     |  66 ++++-
 .../server/controller/KerberosHelperTest.java   | 268 ++++++++++++++++++-
 5 files changed, 435 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a116498e/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
index 9b744d0..980a40f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
@@ -18,6 +18,7 @@
 
 package org.apache.ambari.server.api.resources;
 
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -75,4 +76,13 @@ public class ClusterResourceDefinition extends BaseStacksResourceDefinition {
 
     return setChildren;
   }
+
+  @Override
+  public Collection<String> getUpdateDirectives() {
+    Collection<String> directives = super.getUpdateDirectives();
+    directives.add("regenerate_keytabs");
+
+    return directives;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/a116498e/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index e867f99..b6dd5c4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -1153,12 +1153,12 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       //
       // ***************************************************
 
-      response = updateCluster(request);
+      response = updateCluster(request, requestProperties);
     }
     return response;
   }
 
-  private synchronized RequestStatusResponse updateCluster(ClusterRequest request)
+  private synchronized RequestStatusResponse updateCluster(ClusterRequest request, Map<String, String> requestProperties)
       throws AmbariException {
 
     RequestStageContainer requestStageContainer = null;
@@ -1331,7 +1331,13 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
 
     // set the new security type of the cluster if change is requested
     SecurityType securityType = request.getSecurityType();
-    if((securityType != null) && (cluster.getSecurityType() != securityType) ){
+
+    if(securityType != null) {
+      // if any custom operations are valid and requested, the process of executing them should be initiated,
+      // most of the validation logic will be left to the KerberosHelper to avoid polluting the controller
+      if (kerberosHelper.shouldExecuteCustomOperations(securityType, requestProperties)) {
+        requestStageContainer = kerberosHelper.executeCustomOperations(cluster, request.getKerberosDescriptor(), requestProperties, requestStageContainer);
+      } else if (cluster.getSecurityType() != securityType) {
         LOG.info("Received cluster security type change request from {} to {}",
             cluster.getSecurityType().name(), securityType.name());
 
@@ -1345,6 +1351,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
         }
 
         cluster.setSecurityType(securityType);
+      }
     }
 
     if (requestStageContainer != null) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/a116498e/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index 6bb9bf1..fd1fb57 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -136,6 +136,7 @@ public class KerberosHelper {
    */
   private static ClusterController clusterController = null;
 
+
   /**
    * The Handler implementation that provides the logic to enable Kerberos
    */
@@ -189,11 +190,6 @@ public class KerberosHelper {
     // Update KerberosDetails with the new security type - the current one in the cluster is the "old" value
     kerberosDetails.setSecurityType(securityType);
 
-    //todo: modify call from cluster state transition to not include descriptor
-    if (kerberosDescriptor == null) {
-      kerberosDescriptor = getClusterDescriptor(cluster);
-    }
-
     if (securityType == SecurityType.KERBEROS) {
       LOG.info("Configuring Kerberos for realm {} on cluster, {}", kerberosDetails.getDefaultRealm(), cluster.getClusterName());
       requestStageContainer = handle(cluster, kerberosDescriptor, kerberosDetails, null, null, requestStageContainer, enableKerberosHandler);
@@ -208,6 +204,52 @@ public class KerberosHelper {
   }
 
   /**
+   * Used to execute custom security operations which are sent as directives in URI
+   *
+   * @param cluster               the relevant Cluster
+   * @param kerberosDescriptor    a KerberosDescriptor containing updates to the descriptor already
+   *                              configured for the cluster
+   * @param requestProperties     this structure is expected to hold already supported and validated directives
+   *                              for the 'Cluster' resource. See ClusterResourceDefinition#getUpdateDirectives
+   * @param requestStageContainer a RequestStageContainer to place generated stages, if needed -
+   *                              if null a new RequestStageContainer will be created.   @return the updated or a new RequestStageContainer containing the stages that need to be
+   *                              executed to complete this task; or null if no stages need to be executed.
+   * @throws AmbariException
+   */
+  public RequestStageContainer executeCustomOperations(Cluster cluster, KerberosDescriptor kerberosDescriptor,
+                                                       Map<String, String> requestProperties, RequestStageContainer requestStageContainer)
+      throws AmbariException {
+
+    if (requestProperties != null) {
+
+      for (SupportedCustomOperation operation : SupportedCustomOperation.values()) {
+        if (requestProperties.containsKey(operation.name().toLowerCase())) {
+          String value = requestProperties.get(operation.name().toLowerCase());
+
+          // The operation specific logic is kept in one place and described here
+          switch (operation) {
+            case REGENERATE_KEYTABS:
+              if (cluster.getSecurityType() != SecurityType.KERBEROS) {
+                throw new AmbariException(String.format("Custom operation %s can only be requested with the security type cluster property: %s", operation.name(), SecurityType.KERBEROS.name()));
+              }
+
+              if ("true".equalsIgnoreCase(value)) {
+                handle(cluster, kerberosDescriptor, getKerberosDetails(cluster), null, null, requestStageContainer, createPrincipalsAndKeytabsHandler);
+              }
+              break;
+
+            default: // No other operations are currently supported
+              throw new AmbariException(String.format("Custom operation not supported: %s", operation.name()));
+          }
+        }
+      }
+    }
+
+    return requestStageContainer;
+  }
+
+
+  /**
    * Ensures the set of filtered principals and keytabs exist on the cluster.
    * <p/>
    * No configurations will be altered as a result of this operation, however principals and keytabs
@@ -279,6 +321,11 @@ public class KerberosHelper {
 
     Map<String, Service> services = cluster.getServices();
 
+    //todo: modify call from cluster state transition to not include descriptor
+    if (kerberosDescriptor == null) {
+      kerberosDescriptor = getClusterDescriptor(cluster);
+    }
+
     if ((services != null) && !services.isEmpty()) {
       SecurityState desiredSecurityState = handler.getNewServiceSecurityState();
       String clusterName = cluster.getClusterName();
@@ -696,8 +743,7 @@ public class KerberosHelper {
    * is available from the endpoint
    * stacks/:stackName/versions/:version/artifacts/kerberos_descriptor.
    *
-   * @param cluster  cluster instance
-   *
+   * @param cluster cluster instance
    * @return the kerberos descriptor associated with the specified cluster
    * @throws AmbariException if unable to obtain the descriptor
    */
@@ -733,11 +779,11 @@ public class KerberosHelper {
       // parent cluster doesn't exist.  shouldn't happen since we have the cluster instance
       e.printStackTrace();
       throw new AmbariException("An unknown error occurred while trying to obtain the cluster kerberos descriptor", e);
-    }  catch (NoSuchResourceException e) {
+    } catch (NoSuchResourceException e) {
       // no descriptor registered, use the default from the stack
     }
 
-    if (response != null && ! response.isEmpty()) {
+    if (response != null && !response.isEmpty()) {
       Resource descriptorResource = response.iterator().next();
       String descriptor_data = (String) descriptorResource.getPropertyValue(
           ArtifactResourceProvider.ARTIFACT_DATA_PROPERTY);
@@ -1165,7 +1211,7 @@ public class KerberosHelper {
   /**
    * Determine if a cluster has kerberos enabled.
    *
-   * @param cluster  cluster to test
+   * @param cluster cluster to test
    * @return true if the provided cluster has kerberos enabled; false otherwise
    */
   public boolean isClusterKerberosEnabled(Cluster cluster) {
@@ -1173,6 +1219,13 @@ public class KerberosHelper {
   }
 
   /**
+   * A enumeration of the supported custom operations
+   */
+  public static enum SupportedCustomOperation {
+    REGENERATE_KEYTABS
+  }
+
+  /**
    * Handler is an interface that needs to be implemented by toggle handler classes to do the
    * "right" thing for the task at hand.
    */
@@ -1620,6 +1673,29 @@ public class KerberosHelper {
     }
   }
 
+  /**
+   * Method used to externally peek if weather we have supported operations to execute or not
+   * <p/>
+   * It is required that the SecurityType from the request is wither KERBEROS or NONE and that at least one
+   * directive in the requestProperties map is supported.
+   *
+   * @param requestSecurityType      the SecurityType from the request
+   * @param requestProperties A Map of request directives and their values
+   * @return true if custom operations should be executed; false otherwise
+   */
+  public boolean shouldExecuteCustomOperations(SecurityType requestSecurityType, Map<String, String> requestProperties) {
+
+    if (((requestSecurityType == SecurityType.KERBEROS) || (requestSecurityType == SecurityType.NONE)) &&
+        (requestProperties != null) && !requestProperties.isEmpty()) {
+      for (SupportedCustomOperation type : SupportedCustomOperation.values()) {
+        if (requestProperties.containsKey(type.name().toLowerCase())) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
 
   /**
    * KerberosDetails is a helper class to hold the details of the relevant Kerberos-specific

http://git-wip-us.apache.org/repos/asf/ambari/blob/a116498e/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index 1a66cd9..e0667da 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -536,6 +536,55 @@ public class AmbariManagementControllerImplTest {
   }
 
   /**
+   * Ensure that when the appropriate request directives are set, KerberosHelper#executeCustomOperations
+   * is invoked
+   */
+  @Test
+  public void testUpdateClustersKerberosCustomOperationsInvoked() throws Exception {
+    // member state mocks
+    Capture<AmbariManagementController> controllerCapture = new Capture<AmbariManagementController>();
+    Injector injector = createStrictMock(Injector.class);
+    Cluster cluster = createNiceMock(Cluster.class);
+    ActionManager actionManager = createNiceMock(ActionManager.class);
+    ClusterRequest clusterRequest = createNiceMock(ClusterRequest.class);
+
+    // requests
+    Set<ClusterRequest> requests = Collections.singleton(clusterRequest);
+
+    // request properties (aka directives)
+    Map<String, String> requestProperties = Collections.singletonMap("regenerate_keytabs", "true");
+
+    KerberosHelper kerberosHelper = createMockBuilder(KerberosHelper.class)
+    .addMockedMethod("executeCustomOperations", Cluster.class, KerberosDescriptor.class, Map.class, RequestStageContainer.class)
+    .createStrictMock();
+
+    // expectations
+    injector.injectMembers(capture(controllerCapture));
+    expect(injector.getInstance(Gson.class)).andReturn(null);
+    expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null);
+    expect(injector.getInstance(KerberosHelper.class)).andReturn(kerberosHelper);
+    expect(clusterRequest.getClusterId()).andReturn(1L).times(6);
+    expect(clusterRequest.getSecurityType()).andReturn(SecurityType.KERBEROS).anyTimes();
+    expect(clusters.getClusterById(1L)).andReturn(cluster).times(2);
+    expect(cluster.getClusterName()).andReturn("cluster").times(2);
+
+    expect(kerberosHelper.executeCustomOperations(cluster, null, requestProperties, null))
+        .andReturn(null)
+        .once();
+
+    // replay mocks
+    replay(actionManager, cluster, clusters, injector, clusterRequest, sessionManager, kerberosHelper);
+
+    // test
+    AmbariManagementController controller = new AmbariManagementControllerImpl(actionManager, clusters, injector);
+    controller.updateClusters(requests, requestProperties);
+
+    // assert and verify
+    assertSame(controller, controllerCapture.getValue());
+    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager, kerberosHelper);
+  }
+
+  /**
    * Ensure that when the cluster is updated KerberosHandler.toggleKerberos is not invoked unless
    * the security type is altered
    */
@@ -573,7 +622,7 @@ public class AmbariManagementControllerImplTest {
 
     // assert and verify
     assertSame(controller, controllerCapture.getValue());
-    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager);
+    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager, kerberosHelper);
   }
 
   /**
@@ -607,6 +656,9 @@ public class AmbariManagementControllerImplTest {
     cluster.addSessionAttributes(anyObject(Map.class));
     expectLastCall().once();
 
+    expect(kerberosHelper.shouldExecuteCustomOperations(SecurityType.KERBEROS, null))
+        .andReturn(false)
+        .once();
     expect(kerberosHelper.toggleKerberos(anyObject(Cluster.class), anyObject(SecurityType.class), anyObject(KerberosDescriptor.class), anyObject(RequestStageContainer.class)))
         .andReturn(null)
         .once();
@@ -620,7 +672,7 @@ public class AmbariManagementControllerImplTest {
 
     // assert and verify
     assertSame(controller, controllerCapture.getValue());
-    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager);
+    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager, kerberosHelper);
   }
 
   /**
@@ -654,6 +706,9 @@ public class AmbariManagementControllerImplTest {
     cluster.addSessionAttributes(anyObject(Map.class));
     expectLastCall().once();
 
+    expect(kerberosHelper.shouldExecuteCustomOperations(SecurityType.NONE, null))
+        .andReturn(false)
+        .once();
     expect(kerberosHelper.toggleKerberos(anyObject(Cluster.class), anyObject(SecurityType.class), anyObject(KerberosDescriptor.class), anyObject(RequestStageContainer.class)))
         .andReturn(null)
         .once();
@@ -667,7 +722,7 @@ public class AmbariManagementControllerImplTest {
 
     // assert and verify
     assertSame(controller, controllerCapture.getValue());
-    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager);
+    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager, kerberosHelper);
   }
 
   /**
@@ -710,6 +765,9 @@ public class AmbariManagementControllerImplTest {
     cluster.addSessionAttributes(anyObject(Map.class));
     expectLastCall().once();
 
+    expect(kerberosHelper.shouldExecuteCustomOperations(SecurityType.NONE, null))
+        .andReturn(false)
+        .once();
     expect(kerberosHelper.toggleKerberos(anyObject(Cluster.class), anyObject(SecurityType.class), anyObject(KerberosDescriptor.class), anyObject(RequestStageContainer.class)))
         .andThrow(new IllegalArgumentException("bad args!"))
         .once();
@@ -730,7 +788,7 @@ public class AmbariManagementControllerImplTest {
 
     // assert and verify
     assertSame(controller, controllerCapture.getValue());
-    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager);
+    verify(actionManager, cluster, clusters, injector, clusterRequest, sessionManager, kerberosHelper);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/a116498e/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
index e976d81..3532e69 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -109,7 +109,7 @@ import static org.powermock.api.easymock.PowerMock.verifyAll;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(KerberosDescriptor.class)
-@PowerMockIgnore({"javax.crypto.*" })
+@PowerMockIgnore({"javax.crypto.*"})
 @SuppressWarnings("unchecked")
 public class KerberosHelperTest {
 
@@ -309,6 +309,35 @@ public class KerberosHelperTest {
     }
   }
 
+  @Test
+  public void testExecuteCustomOperationsInvalidOperation() throws Exception {
+    KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
+    final Cluster cluster = createNiceMock(Cluster.class);
+
+    try {
+      kerberosHelper.executeCustomOperations(cluster, createNiceMock(KerberosDescriptor.class),
+          Collections.singletonMap("invalid_operation", "false"), null);
+    }
+    catch(Throwable t) {
+      Assert.fail("Exception should not have been thrown");
+    }
+  }
+
+  @Test(expected = AmbariException.class)
+  public void testRegenerateKeytabsInvalidValue() throws Exception {
+    KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
+    final Cluster cluster = createNiceMock(Cluster.class);
+
+    kerberosHelper.executeCustomOperations(cluster, createNiceMock(KerberosDescriptor.class),
+        Collections.singletonMap("regenerate_keytabs", "false"), null);
+    Assert.fail("AmbariException should have failed");
+  }
+
+  @Test
+  public void testRegenerateKeytabs() throws Exception {
+    testRegenerateKeytabs(new KerberosCredential("principal", "password", "keytab"), false, false);
+  }
+
   private void testEnableKerberos(final KerberosCredential kerberosCredential,
                                   boolean getClusterDescriptor,
                                   boolean getStackDescriptor) throws Exception {
@@ -548,6 +577,243 @@ public class KerberosHelperTest {
     verifyAll();
   }
 
+  private void testRegenerateKeytabs(final KerberosCredential kerberosCredential,
+                                     boolean getClusterDescriptor,
+                                     boolean getStackDescriptor) throws Exception {
+
+    KerberosHelper kerberosHelper = injector.getInstance(KerberosHelper.class);
+
+    final StackId stackVersion = createNiceMock(StackId.class);
+
+    final ServiceComponentHost sch1 = createMock(ServiceComponentHost.class);
+    expect(sch1.getServiceName()).andReturn("SERVICE1").once();
+    expect(sch1.getServiceComponentName()).andReturn("COMPONENT1").once();
+    expect(sch1.getSecurityState()).andReturn(SecurityState.UNSECURED).anyTimes();
+    expect(sch1.getDesiredSecurityState()).andReturn(SecurityState.UNSECURED).anyTimes();
+    expect(sch1.getStackVersion()).andReturn(stackVersion).anyTimes();
+    expect(sch1.getHostName()).andReturn("host1").anyTimes();
+
+    final ServiceComponentHost sch2 = createMock(ServiceComponentHost.class);
+    expect(sch2.getServiceName()).andReturn("SERVICE2").once();
+    expect(sch2.getServiceComponentName()).andReturn("COMPONENT2").once();
+    expect(sch2.getSecurityState()).andReturn(SecurityState.UNSECURED).anyTimes();
+    expect(sch2.getDesiredSecurityState()).andReturn(SecurityState.UNSECURED).anyTimes();
+    expect(sch2.getStackVersion()).andReturn(stackVersion).anyTimes();
+    expect(sch2.getHostName()).andReturn("host1").anyTimes();
+
+    final Host host = createNiceMock(Host.class);
+    expect(host.getHostName()).andReturn("host1").once();
+    expect(host.getState()).andReturn(HostState.HEALTHY).once();
+
+    final Service service1 = createStrictMock(Service.class);
+    expect(service1.getName()).andReturn("SERVICE1").anyTimes();
+    expect(service1.getServiceComponents())
+        .andReturn(Collections.<String, ServiceComponent>emptyMap())
+        .once();
+
+    final Service service2 = createStrictMock(Service.class);
+    expect(service2.getName()).andReturn("SERVICE2").anyTimes();
+    expect(service2.getServiceComponents())
+        .andReturn(Collections.<String, ServiceComponent>emptyMap())
+        .once();
+
+    final Map<String, String> kerberosEnvProperties = createNiceMock(Map.class);
+    // TODO: (rlevas) Add when AMBARI 9121 is complete
+    // expect(kerberosEnvProperties.get("kdc_type")).andReturn("mit-kdc").once();
+
+    final Config kerberosEnvConfig = createNiceMock(Config.class);
+    expect(kerberosEnvConfig.getProperties()).andReturn(kerberosEnvProperties).once();
+
+    final Map<String, String> krb5ConfProperties = createNiceMock(Map.class);
+    expect(krb5ConfProperties.get("kdc_type")).andReturn("mit-kdc").once();
+    expect(krb5ConfProperties.get("realm")).andReturn("FOOBAR.COM").once();
+
+    final Config krb5ConfConfig = createNiceMock(Config.class);
+    // TODO: (rlevas) Remove when AMBARI 9121 is complete
+    expect(krb5ConfConfig.getProperties()).andReturn(krb5ConfProperties).once();
+
+    final MaintenanceStateHelper maintenanceStateHelper = injector.getInstance(MaintenanceStateHelper.class);
+    expect(maintenanceStateHelper.getEffectiveState(anyObject(ServiceComponentHost.class)))
+        .andReturn(MaintenanceState.OFF).anyTimes();
+
+    final Cluster cluster = createNiceMock(Cluster.class);
+    expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).once();
+    expect(cluster.getDesiredConfigByType("krb5-conf")).andReturn(krb5ConfConfig).once();
+    expect(cluster.getDesiredConfigByType("kerberos-env")).andReturn(kerberosEnvConfig).once();
+    expect(cluster.getClusterName()).andReturn("c1").anyTimes();
+    expect(cluster.getServices())
+        .andReturn(new HashMap<String, Service>() {
+          {
+            put("SERVICE1", service1);
+            put("SERVICE2", service2);
+          }
+        })
+        .anyTimes();
+    expect(cluster.getServiceComponentHosts("host1"))
+        .andReturn(new ArrayList<ServiceComponentHost>() {
+          {
+            add(sch1);
+            add(sch2);
+          }
+        })
+        .once();
+    expect(cluster.getCurrentStackVersion())
+        .andReturn(new StackId("HDP", "2.2"))
+        .anyTimes();
+    expect(cluster.getSessionAttributes()).andReturn(new HashMap<String, Object>() {{
+      if (kerberosCredential != null) {
+        put("kerberos_admin/" + KerberosCredential.KEY_NAME_PRINCIPAL, kerberosCredential.getPrincipal());
+        put("kerberos_admin/" + KerberosCredential.KEY_NAME_PASSWORD, kerberosCredential.getPassword());
+        put("kerberos_admin/" + KerberosCredential.KEY_NAME_KEYTAB, kerberosCredential.getKeytab());
+      }
+    }}).anyTimes();
+
+    final Clusters clusters = injector.getInstance(Clusters.class);
+    expect(clusters.getHostsForCluster("c1"))
+        .andReturn(new HashMap<String, Host>() {
+          {
+            put("host1", host);
+          }
+        })
+        .once();
+
+    final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1"))
+        .andReturn(Collections.<String, Map<String, String>>emptyMap())
+        .once();
+    expect(ambariManagementController.getRoleCommandOrder(cluster))
+        .andReturn(createNiceMock(RoleCommandOrder.class))
+        .once();
+
+    final ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
+    expect(configHelper.getEffectiveConfigProperties(anyObject(Cluster.class), anyObject(Map.class)))
+        .andReturn(new HashMap<String, Map<String, String>>() {
+          {
+            put("cluster-env", new HashMap<String, String>() {{
+              put("kerberos_domain", "FOOBAR.COM");
+            }});
+          }
+        })
+        .once();
+    expect(configHelper.getEffectiveConfigAttributes(anyObject(Cluster.class), anyObject(Map.class)))
+        .andReturn(Collections.<String, Map<String, Map<String, String>>>emptyMap())
+        .once();
+
+    final KerberosPrincipalDescriptor principalDescriptor1 = createNiceMock(KerberosPrincipalDescriptor.class);
+    expect(principalDescriptor1.getValue()).andReturn("component1/_HOST@${realm}").once();
+    expect(principalDescriptor1.getType()).andReturn(KerberosPrincipalType.SERVICE).once();
+    expect(principalDescriptor1.getConfiguration()).andReturn("service1-site/component1.kerberos.principal").once();
+
+    final KerberosPrincipalDescriptor principalDescriptor2 = createNiceMock(KerberosPrincipalDescriptor.class);
+    expect(principalDescriptor2.getValue()).andReturn("component2/${host}@${realm}").once();
+    expect(principalDescriptor2.getType()).andReturn(KerberosPrincipalType.SERVICE).once();
+    expect(principalDescriptor2.getConfiguration()).andReturn("service2-site/component2.kerberos.principal").once();
+
+    final KerberosKeytabDescriptor keytabDescriptor1 = createNiceMock(KerberosKeytabDescriptor.class);
+    expect(keytabDescriptor1.getFile()).andReturn("${keytab_dir}/service1.keytab").once();
+    expect(keytabDescriptor1.getOwnerName()).andReturn("service1").once();
+    expect(keytabDescriptor1.getOwnerAccess()).andReturn("rw").once();
+    expect(keytabDescriptor1.getGroupName()).andReturn("hadoop").once();
+    expect(keytabDescriptor1.getGroupAccess()).andReturn("").once();
+    expect(keytabDescriptor1.getConfiguration()).andReturn("service1-site/component1.keytab.file").once();
+
+    final KerberosKeytabDescriptor keytabDescriptor2 = createNiceMock(KerberosKeytabDescriptor.class);
+    expect(keytabDescriptor2.getFile()).andReturn("${keytab_dir}/service2.keytab").once();
+    expect(keytabDescriptor2.getOwnerName()).andReturn("service2").once();
+    expect(keytabDescriptor2.getOwnerAccess()).andReturn("rw").once();
+    expect(keytabDescriptor2.getGroupName()).andReturn("hadoop").once();
+    expect(keytabDescriptor2.getGroupAccess()).andReturn("").once();
+    expect(keytabDescriptor2.getConfiguration()).andReturn("service2-site/component2.keytab.file").once();
+
+    final KerberosIdentityDescriptor identityDescriptor1 = createNiceMock(KerberosIdentityDescriptor.class);
+    expect(identityDescriptor1.getPrincipalDescriptor()).andReturn(principalDescriptor1).once();
+    expect(identityDescriptor1.getKeytabDescriptor()).andReturn(keytabDescriptor1).once();
+
+    final KerberosIdentityDescriptor identityDescriptor2 = createNiceMock(KerberosIdentityDescriptor.class);
+    expect(identityDescriptor2.getPrincipalDescriptor()).andReturn(principalDescriptor2).once();
+    expect(identityDescriptor2.getKeytabDescriptor()).andReturn(keytabDescriptor2).once();
+
+    final KerberosComponentDescriptor componentDescriptor1 = createNiceMock(KerberosComponentDescriptor.class);
+    expect(componentDescriptor1.getIdentities(true)).
+        andReturn(new ArrayList<KerberosIdentityDescriptor>() {{
+          add(identityDescriptor1);
+        }}).once();
+
+    final KerberosComponentDescriptor componentDescriptor2 = createNiceMock(KerberosComponentDescriptor.class);
+    expect(componentDescriptor2.getIdentities(true)).
+        andReturn(new ArrayList<KerberosIdentityDescriptor>() {{
+          add(identityDescriptor2);
+        }}).once();
+
+    final KerberosServiceDescriptor serviceDescriptor1 = createNiceMock(KerberosServiceDescriptor.class);
+    expect(serviceDescriptor1.getComponent("COMPONENT1")).andReturn(componentDescriptor1).once();
+
+    final KerberosServiceDescriptor serviceDescriptor2 = createNiceMock(KerberosServiceDescriptor.class);
+    expect(serviceDescriptor2.getComponent("COMPONENT2")).andReturn(componentDescriptor2).once();
+
+    final KerberosDescriptor kerberosDescriptor = createNiceMock(KerberosDescriptor.class);
+    expect(kerberosDescriptor.getService("SERVICE1")).andReturn(serviceDescriptor1).once();
+    expect(kerberosDescriptor.getService("SERVICE2")).andReturn(serviceDescriptor2).once();
+
+    //todo: extract method?
+    if (getClusterDescriptor) {
+      // needed to mock the static method fromJson()
+      setupGetDescriptorFromCluster(kerberosDescriptor);
+    } else if (getStackDescriptor) {
+      setupGetDescriptorFromStack(kerberosDescriptor);
+    }
+    final StageFactory stageFactory = injector.getInstance(StageFactory.class);
+    expect(stageFactory.createNew(anyLong(), anyObject(String.class), anyObject(String.class),
+        anyLong(), anyObject(String.class), anyObject(String.class), anyObject(String.class),
+        anyObject(String.class)))
+        .andAnswer(new IAnswer<Stage>() {
+          @Override
+          public Stage answer() throws Throwable {
+            Stage stage = createNiceMock(Stage.class);
+
+            expect(stage.getHostRoleCommands())
+                .andReturn(Collections.<String, Map<String, HostRoleCommand>>emptyMap())
+                .anyTimes();
+            replay(stage);
+            return stage;
+          }
+        })
+        .anyTimes();
+
+    // This is a STRICT mock to help ensure that the end result is what we want.
+    final RequestStageContainer requestStageContainer = createStrictMock(RequestStageContainer.class);
+    // Create Principals Stage
+    expect(requestStageContainer.getLastStageId()).andReturn(-1L).anyTimes();
+    expect(requestStageContainer.getId()).andReturn(1L).once();
+    requestStageContainer.addStages(anyObject(List.class));
+    expectLastCall().once();
+    // Create Keytabs Stage
+    expect(requestStageContainer.getLastStageId()).andReturn(0L).anyTimes();
+    expect(requestStageContainer.getId()).andReturn(1L).once();
+    requestStageContainer.addStages(anyObject(List.class));
+    expectLastCall().once();
+    // Distribute Keytabs Stage
+    expect(requestStageContainer.getLastStageId()).andReturn(1L).anyTimes();
+    expect(requestStageContainer.getId()).andReturn(1L).once();
+    requestStageContainer.addStages(anyObject(List.class));
+    expectLastCall().once();
+    // Clean-up/Finalize Stage
+    expect(requestStageContainer.getLastStageId()).andReturn(3L).anyTimes();
+    expect(requestStageContainer.getId()).andReturn(1L).once();
+    requestStageContainer.addStages(anyObject(List.class));
+    expectLastCall().once();
+
+    replayAll();
+
+    // Needed by infrastructure
+    metaInfo.init();
+
+    kerberosHelper.executeCustomOperations(cluster, !(getClusterDescriptor || getStackDescriptor) ?
+        kerberosDescriptor : null, Collections.singletonMap("regenerate_keytabs", "true"), requestStageContainer);
+
+    verifyAll();
+  }
+
   private void setupGetDescriptorFromCluster(KerberosDescriptor kerberosDescriptor) throws Exception {
     mockStatic(KerberosDescriptor.class);
     ResourceProvider resourceProvider = createStrictMock(ResourceProvider.class);