You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/08/29 13:25:19 UTC
[16/50] [abbrv] ambari git commit: AMBARI-21602. Pre-configure
services when Kerberos is enabled to reduce number of core service restarts
when services are added (rlevas)
AMBARI-21602. Pre-configure services when Kerberos is enabled to reduce number of core service restarts when services are added (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4bce5782
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4bce5782
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4bce5782
Branch: refs/heads/feature-branch-AMBARI-21307
Commit: 4bce57823da7d972df9e422e3f920ebbc60362f3
Parents: f5f9a75
Author: Robert Levas <rl...@hortonworks.com>
Authored: Thu Aug 24 17:01:12 2017 -0400
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Thu Aug 24 17:01:12 2017 -0400
----------------------------------------------------------------------
.../server/api/services/AmbariMetaInfo.java | 115 +--
.../stackadvisor/StackAdvisorRequest.java | 7 +-
.../controller/DeleteIdentityHandler.java | 2 +-
.../server/controller/KerberosHelper.java | 114 ++-
.../server/controller/KerberosHelperImpl.java | 818 +++++++++++++------
.../BlueprintConfigurationProcessor.java | 2 +-
...usterKerberosDescriptorResourceProvider.java | 3 +-
.../internal/StackVersionResourceProvider.java | 66 --
.../utilities/RemovableIdentities.java | 4 +-
.../controller/utilities/UsedIdentities.java | 2 +-
.../customactions/ActionDefinitionManager.java | 4 +-
.../AbstractPrepareKerberosServerAction.java | 2 +-
.../kerberos/KerberosServerAction.java | 6 +
.../kerberos/PreconfigureServiceType.java | 46 ++
.../PrepareDisableKerberosServerAction.java | 2 +-
.../PrepareEnableKerberosServerAction.java | 38 +-
.../PrepareKerberosIdentitiesServerAction.java | 48 +-
.../upgrades/UpgradeUserKerberosDescriptor.java | 4 +-
.../server/stack/ConfigurationDirectory.java | 5 +-
.../ambari/server/stack/ExtensionDirectory.java | 3 +-
.../ambari/server/stack/ServiceDirectory.java | 14 +-
.../ambari/server/stack/ServiceModule.java | 6 +-
.../ambari/server/stack/StackDirectory.java | 92 ++-
.../ambari/server/stack/StackManager.java | 13 +-
.../apache/ambari/server/stack/StackModule.java | 9 +-
.../ambari/server/state/ConfigHelper.java | 3 +-
.../apache/ambari/server/state/ServiceInfo.java | 11 +-
.../apache/ambari/server/state/StackInfo.java | 20 +
.../kerberos/AbstractKerberosDescriptor.java | 36 +-
.../state/kerberos/KerberosDescriptor.java | 42 +-
.../kerberos/KerberosServiceDescriptor.java | 86 +-
.../server/upgrade/AbstractUpgradeCatalog.java | 2 +-
.../1.10.3-10/configuration/kerberos-env.xml | 27 +
.../stacks/HDP/2.6/kerberos_preconfigure.json | 24 +
.../render/ClusterBlueprintRendererTest.java | 2 +-
.../server/api/services/AmbariMetaInfoTest.java | 55 +-
.../server/controller/KerberosHelperTest.java | 707 ++++++++++++----
.../BlueprintConfigurationProcessorTest.java | 2 +-
...rKerberosDescriptorResourceProviderTest.java | 14 +-
.../utilities/KerberosIdentityCleanerTest.java | 2 +-
.../UpgradeUserKerberosDescriptorTest.java | 6 +-
.../ambari/server/stack/ServiceModuleTest.java | 3 +-
.../KerberosDescriptorUpdateHelperTest.java | 4 +-
.../kerberos/KerberosServiceDescriptorTest.java | 2 +
.../stacks/HDP/2.0.8/kerberos_preconfigure.json | 23 +
45 files changed, 1780 insertions(+), 716 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 7eb82e4..aad35a9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -24,7 +24,6 @@ import static org.apache.ambari.server.controller.utilities.PropertyHelper.AGGRE
import java.io.File;
import java.io.FileReader;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
@@ -55,7 +54,6 @@ import org.apache.ambari.server.metadata.AmbariServiceAlertDefinitions;
import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
import org.apache.ambari.server.orm.dao.MetainfoDAO;
import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
-import org.apache.ambari.server.stack.StackDirectory;
import org.apache.ambari.server.stack.StackManager;
import org.apache.ambari.server.stack.StackManagerFactory;
import org.apache.ambari.server.state.Cluster;
@@ -97,30 +95,6 @@ import com.google.inject.Singleton;
*/
@Singleton
public class AmbariMetaInfo {
- public static final String SERVICE_CONFIG_FOLDER_NAME = "configuration";
- public static final String SERVICE_PROPERTIES_FOLDER_NAME = "properties";
- public static final String SERVICE_THEMES_FOLDER_NAME = "themes";
- public static final String SERVICE_QUICKLINKS_CONFIGURATIONS_FOLDER_NAME = "quicklinks";
- public static final String SERVICE_CONFIG_FILE_NAME_POSTFIX = ".xml";
- public static final String RCO_FILE_NAME = "role_command_order.json";
- public static final String SERVICE_METRIC_FILE_NAME = "metrics.json";
- public static final String SERVICE_ALERT_FILE_NAME = "alerts.json";
- public static final String SERVICE_ADVISOR_FILE_NAME = "service_advisor.py";
-
- /**
- * The filename for a Kerberos descriptor file at either the stack or service level
- */
- public static final String KERBEROS_DESCRIPTOR_FILE_NAME = "kerberos.json";
-
- /**
- * The filename for a Widgets descriptor file at either the stack or service level
- */
- public static final String WIDGETS_DESCRIPTOR_FILE_NAME = "widgets.json";
-
- /**
- * Filename for theme file at service layer
- */
- public static final String SERVICE_THEME_FILE_NAME = "theme.json";
/**
* This string is used in placeholder in places that are common for
@@ -131,13 +105,6 @@ public class AmbariMetaInfo {
* Version of XML files with support of custom services and custom commands
*/
public static final String SCHEMA_VERSION_2 = "2.0";
- public static final FilenameFilter FILENAME_FILTER = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String s) {
- return !(s.equals(".svn") || s.equals(".git") ||
- s.equals(StackDirectory.HOOKS_FOLDER_NAME));
- }
- };
private final static Logger LOG = LoggerFactory.getLogger(AmbariMetaInfo.class);
@@ -1276,40 +1243,52 @@ public class AmbariMetaInfo {
* All of the kerberos.json files from the specified stack (and version) are read, parsed and
* complied into a complete Kerberos descriptor hierarchy.
*
- * @param stackName a String declaring the stack name
- * @param stackVersion a String declaring the stack version
+ * @param stackName a String declaring the stack name
+ * @param stackVersion a String declaring the stack version
+ * @param includePreconfigureData a Boolean value indicating whether to include the pre-configuration
+ * data (<code>true</code>), or not (<code>false</code>)
* @return a new complete KerberosDescriptor, or null if no Kerberos descriptor information is available
* @throws AmbariException if an error occurs reading or parsing the stack's kerberos.json files
*/
- public KerberosDescriptor getKerberosDescriptor(String stackName, String stackVersion) throws AmbariException {
+ public KerberosDescriptor getKerberosDescriptor(String stackName, String stackVersion, boolean includePreconfigureData) throws AmbariException {
StackInfo stackInfo = getStack(stackName, stackVersion);
- String kerberosDescriptorFileLocation = stackInfo.getKerberosDescriptorFileLocation();
-
KerberosDescriptor kerberosDescriptor = null;
- // Read in the stack-level Kerberos descriptor
- if (kerberosDescriptorFileLocation != null) {
- File file = new File(kerberosDescriptorFileLocation);
+ // Read in the stack-level Kerberos descriptor pre-configuration data
+ if (includePreconfigureData) {
+ kerberosDescriptor = readKerberosDescriptorFromFile(stackInfo.getKerberosDescriptorPreConfigurationFileLocation());
- if (file.canRead()) {
- try {
- kerberosDescriptor = kerberosDescriptorFactory.createInstance(file);
- } catch (IOException e) {
- throw new AmbariException(String.format("Failed to parse kerberos descriptor file %s",
- file.getAbsolutePath()), e);
+ if (kerberosDescriptor != null) {
+ // Ensure the all services to be pre-configured are flagged appropriately.
+ Map<String, KerberosServiceDescriptor> serviceDescriptors = kerberosDescriptor.getServices();
+ if (serviceDescriptors != null) {
+ for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) {
+ serviceDescriptor.setPreconfigure(true);
+ }
}
- } else {
- throw new AmbariException(String.format("Unable to read kerberos descriptor file %s",
- file.getAbsolutePath()));
}
}
- if (kerberosDescriptor == null) {
- kerberosDescriptor = new KerberosDescriptor();
+ // Read in the base stack-level Kerberos descriptor.
+ KerberosDescriptor stackKerberosDescriptor = readKerberosDescriptorFromFile(stackInfo.getKerberosDescriptorFileLocation());
+ if (stackKerberosDescriptor == null) {
+ // If kerberosDescriptor is null and stackKerberosDescriptor is null, then ensure
+ // kerberosDescriptor is an empty KerberosDescriptor.
+ if (kerberosDescriptor == null) {
+ kerberosDescriptor = new KerberosDescriptor();
+ }
+ } else {
+ if (kerberosDescriptor == null) {
+ // If kerberosDescriptor is null; then set it to stackKerberosDescriptor.
+ kerberosDescriptor = stackKerberosDescriptor;
+ } else {
+ // If kerberosDescriptor is not null; then update it using stackKerberosDescriptor.
+ kerberosDescriptor.update(stackKerberosDescriptor);
+ }
}
- // Read in the service-level Kerberos descriptors
+ // Read in the individual service-level Kerberos descriptors
Map<String, ServiceInfo> services = getServices(stackName, stackVersion);
if (services != null) {
@@ -1318,6 +1297,9 @@ public class AmbariMetaInfo {
if (serviceDescriptors != null) {
for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors) {
+ // Add the service-level Kerberos descriptor to kerberosDescriptor. If the service was
+ // previously added (possible from the pre-configure data), update the existing service-level
+ // Kerberos descriptor with this one.
kerberosDescriptor.putService(serviceDescriptor);
}
}
@@ -1425,5 +1407,32 @@ public class AmbariMetaInfo {
return versionDefinitions;
}
+ /**
+ * Reads a Kerberos descriptor from the specified file path.
+ *
+ * @param fileLocation the path to the file
+ * @return a KerberosDescriptor or <code>null</code>, if no path is specified
+ * @throws AmbariException if an error occurs reading or parsing the Kerberos descriptor file
+ */
+ KerberosDescriptor readKerberosDescriptorFromFile(String fileLocation) throws AmbariException {
+ if (!StringUtils.isEmpty(fileLocation)) {
+ File file = new File(fileLocation);
+
+ if (file.canRead()) {
+ try {
+ return kerberosDescriptorFactory.createInstance(file);
+ } catch (IOException e) {
+ throw new AmbariException(String.format("Failed to parse Kerberos descriptor file %s",
+ file.getAbsolutePath()), e);
+ }
+ } else {
+ throw new AmbariException(String.format("Unable to read Kerberos descriptor file %s",
+ file.getAbsolutePath()));
+ }
+ } else {
+ LOG.debug("Missing path to Kerberos descriptor, returning null");
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index 7ba1b18..3a2b488 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.api.services.stackadvisor;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -39,7 +40,7 @@ public class StackAdvisorRequest {
private String stackVersion;
private StackAdvisorRequestType requestType;
private List<String> hosts = new ArrayList<>();
- private List<String> services = new ArrayList<>();
+ private Collection<String> services = new ArrayList<>();
private Map<String, Set<String>> componentHostsMap = new HashMap<>();
private Map<String, Set<String>> hostComponents = new HashMap<>();
private Map<String, Set<String>> hostGroupBindings = new HashMap<>();
@@ -64,7 +65,7 @@ public class StackAdvisorRequest {
return hosts;
}
- public List<String> getServices() {
+ public Collection<String> getServices() {
return services;
}
@@ -142,7 +143,7 @@ public class StackAdvisorRequest {
return this;
}
- public StackAdvisorRequestBuilder forServices(List<String> services) {
+ public StackAdvisorRequestBuilder forServices(Collection<String> services) {
this.instance.services = services;
return this;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
index 3329e76..978b329 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
@@ -279,7 +279,7 @@ class DeleteIdentityHandler {
}
private KerberosDescriptor getKerberosDescriptor() throws AmbariException {
- return getKerberosHelper().getKerberosDescriptor(getCluster());
+ return getKerberosHelper().getKerberosDescriptor(getCluster(), false);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4bce5782/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index a334542..9bdb377 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -80,6 +80,10 @@ public interface KerberosHelper {
*/
String AMBARI_SERVER_HOST_NAME = "ambari_server";
/**
+ * The name of the Kerberos environment configuration type
+ */
+ String KERBEROS_ENV = "kerberos-env";
+ /**
* The name of the Ambari server's Kerberos identities as defined in the Kerberos descriptor
*/
String AMBARI_SERVER_KERBEROS_IDENTITY_NAME = "ambari-server";
@@ -93,6 +97,32 @@ public interface KerberosHelper {
* identities or not
*/
String MANAGE_IDENTITIES = "manage_identities";
+ /**
+ * The kerberos-env property name declaring the default realm
+ */
+ String DEFAULT_REALM = "realm";
+ /**
+ * The kerberos-env property name declaring the kdc_type
+ *
+ * @see org.apache.ambari.server.serveraction.kerberos.KDCType
+ */
+ String KDC_TYPE = "kdc_type";
+ /**
+ * The kerberos-env property name declaring whether to manage auth-to-local rules or not
+ */
+ String MANAGE_AUTH_TO_LOCAL_RULES = "manage_auth_to_local";
+ /**
+ * The kerberos-env property name declaring whether auth-to-local rules should be case-insensitive or not
+ */
+ String CASE_INSENSITIVE_USERNAME_RULES = "case_insensitive_username_rules";
+ /**
+ * The kerberos-env property name declaring how to preprocess services.
+ * <p>
+ * Expected values are <code>"ALL"</code>, <code>"DEFAULT"</code>, <code>"NONE"</code>
+ *
+ * @see org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType
+ */
+ String PRECONFIGURE_SERVICES = "preconfigure_services";
/**
* Toggles Kerberos security to enable it or remove it depending on the state of the cluster.
@@ -379,18 +409,22 @@ public interface KerberosHelper {
* Sets the relevant auth-to-local rule configuration properties using the services installed on
* the cluster and their relevant Kerberos descriptors to determine the rules to be created.
*
- * @param kerberosDescriptor the current Kerberos descriptor
- * @param realm the default realm
- * @param installedServices the map of services and relevant components to process
- * @param existingConfigurations a map of the current configurations
- * @param kerberosConfigurations a map of the configurations to update, this where the generated
- * auth-to-local values will be stored
+ * @param cluster cluster instance
+ * @param kerberosDescriptor the current Kerberos descriptor
+ * @param realm the default realm
+ * @param installedServices the map of services and relevant components to process
+ * @param existingConfigurations a map of the current configurations
+ * @param kerberosConfigurations a map of the configurations to update, this where the generated
+ * auth-to-local values will be stored
+ * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
* @throws AmbariException
*/
- void setAuthToLocalRules(KerberosDescriptor kerberosDescriptor, String realm,
+ void setAuthToLocalRules(Cluster cluster, KerberosDescriptor kerberosDescriptor,
+ String realm,
Map<String, Set<String>> installedServices,
Map<String, Map<String, String>> existingConfigurations,
- Map<String, Map<String, String>> kerberosConfigurations)
+ Map<String, Map<String, String>> kerberosConfigurations,
+ boolean includePreconfigureData)
throws AmbariException;
/**
@@ -437,12 +471,13 @@ public interface KerberosHelper {
* descriptor and the composite is returned. If not, the default cluster descriptor is returned
* as-is.
*
- * @param cluster cluster instance
+ * @param cluster cluster instance
+ * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
* @return the kerberos descriptor associated with the specified cluster
* @throws AmbariException if unable to obtain the descriptor
- * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection)
+ * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection, boolean)
*/
- KerberosDescriptor getKerberosDescriptor(Cluster cluster) throws AmbariException;
+ KerberosDescriptor getKerberosDescriptor(Cluster cluster, boolean includePreconfigureData) throws AmbariException;
/**
* Gets the requested Kerberos descriptor.
@@ -456,39 +491,62 @@ public interface KerberosHelper {
* <dd>A Kerberos descriptor built using user-specified data stored as an artifact of the cluster, only</dd>
* <dt>{@link KerberosDescriptorType#COMPOSITE}</dt>
* <dd>A Kerberos descriptor built using data from the current stack definition with user-specified data stored as an artifact of the cluster applied
- * - see {@link #getKerberosDescriptor(Cluster)}</dd>
+ * - see {@link #getKerberosDescriptor(Cluster, boolean)}</dd>
* </dl>
*
- * @param kerberosDescriptorType the type of Kerberos descriptor to retrieve - see {@link KerberosDescriptorType}
- * @param cluster the relevant Cluster
- * @param evaluateWhenClauses true to evaluate Kerberos identity <code>when</code> clauses and
- * prune if necessary; false otherwise.
- * @param additionalServices an optional collection of service names declaring additional
- * services to add to the set of currently installed services to use
- * while evaluating <code>when</code> clauses
+ * @param kerberosDescriptorType the type of Kerberos descriptor to retrieve - see {@link KerberosDescriptorType}
+ * @param cluster the relevant Cluster
+ * @param evaluateWhenClauses true to evaluate Kerberos identity <code>when</code> clauses and
+ * prune if necessary; false otherwise.
+ * @param additionalServices an optional collection of service names declaring additional
+ * services to add to the set of currently installed services to use
+ * while evaluating <code>when</code> clauses
+ * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
* @return a Kerberos descriptor
* @throws AmbariException
*/
KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
- boolean evaluateWhenClauses, Collection<String> additionalServices)
+ boolean evaluateWhenClauses, Collection<String> additionalServices, boolean includePreconfigureData)
throws AmbariException;
/**
- * Merges configuration from a Map of configuration updates into a main configurations Map. Each
- * property in the updates Map is processed to replace variables using the replacement Map.
+ * Merges configurations from a Map of configuration updates into a main configurations Map.
+ * <p>
+ * Each property in the updates Map is processed to replace variables using the replacement Map,
+ * unless it has been filtered out using an optionally supplied configuration type filter
+ * (configurationTypeFilter).
* <p/>
* See {@link org.apache.ambari.server.state.kerberos.VariableReplacementHelper#replaceVariables(String, java.util.Map)}
* for information on variable replacement.
*
- * @param configurations a Map of configurations
- * @param updates a Map of configuration updates
- * @param replacements a Map of (grouped) replacement values
- * @return the merged Map
- * @throws AmbariException
+ * @param configurations a Map of existing configurations (updated in-place)
+ * @param updates a Map of configuration updates
+ * @param replacements a Map of (grouped) replacement values
+ * @param configurationTypeFilter a Set of config types to filter from the map of updates;
+ * <code>null</code> indicate no filter
+ * @return the updated configurations map
+ * @throws AmbariException if an issue occurs
*/
Map<String, Map<String, String>> mergeConfigurations(Map<String, Map<String, String>> configurations,
Map<String, KerberosConfigurationDescriptor> updates,
- Map<String, Map<String, String>> replacements)
+ Map<String, Map<String, String>> replacements,
+ Set<String> configurationTypeFilter)
+ throws AmbariException;
+
+ /**
+ * Determines which services, not currently installed, should be preconfigured to aid in reducing
+ * the number of service restarts when new services are added to a cluster where Kerberos is enabled.
+ *
+ * @param configurations a Map of existing configurations (updated in-place)
+ * @param replacements a Map of (grouped) replacement values
+ * @param cluster the cluster
+ * @param kerberosDescriptor the Kerberos Descriptor
+ * @return the updated configurations map
+ * @throws AmbariException if an issue occurs
+ */
+ Map<String, Map<String, String>> processPreconfiguredServiceConfigurations(Map<String, Map<String, String>> configurations,
+ Map<String, Map<String, String>> replacements, Cluster cluster,
+ KerberosDescriptor kerberosDescriptor)
throws AmbariException;
/**