You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ad...@apache.org on 2018/11/28 07:17:11 UTC
[ambari] branch trunk updated: AMBARI-24947. Support for complex
Add Service request in secure cluster (#2653)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new a6250a9 AMBARI-24947. Support for complex Add Service request in secure cluster (#2653)
a6250a9 is described below
commit a6250a91fcb69af5baa14ea38ed835effc653a89
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Wed Nov 28 08:17:07 2018 +0100
AMBARI-24947. Support for complex Add Service request in secure cluster (#2653)
---
.../server/controller/AddServiceRequest.java | 85 ++++++++++----
.../ambari/server/controller/KerberosHelper.java | 5 +
.../server/controller/KerberosHelperImpl.java | 12 +-
.../internal/ClusterResourceProvider.java | 4 +-
.../internal/ProvisionClusterRequest.java | 13 ++-
.../internal/ServiceResourceProvider.java | 6 +-
.../topology/ClusterConfigurationRequest.java | 123 +++++---------------
.../ambari/server/topology/Configuration.java | 65 +++++++++++
.../apache/ambari/server/topology/Credential.java | 65 ++++++++++-
.../server/topology/SecurityConfiguration.java | 52 ++++++++-
.../ambari/server/topology/TopologyManager.java | 38 +++---
.../server/topology/addservice/AddServiceInfo.java | 9 +-
.../addservice/AddServiceOrchestrator.java | 128 ++++++++++++++++++++-
.../addservice/ResourceProviderAdapter.java | 92 +++++++--------
.../ambari/server/utils/ShellCommandUtil.java | 3 +
.../org/apache/ambari/server/utils/StageUtils.java | 24 ++++
.../server/controller/AddServiceRequestTest.java | 17 +++
.../topology/ClusterConfigurationRequestTest.java | 2 +-
.../test/resources/add_service_api/request1.json | 16 ++-
19 files changed, 545 insertions(+), 214 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
index 61edde3..f3217d1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AddServiceRequest.java
@@ -20,7 +20,10 @@ package org.apache.ambari.server.controller;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Collections.emptySet;
+import static java.util.stream.Collectors.toMap;
import static org.apache.ambari.server.controller.internal.BaseClusterRequest.PROVISION_ACTION_PROPERTY;
+import static org.apache.ambari.server.controller.internal.ClusterResourceProvider.CREDENTIALS;
+import static org.apache.ambari.server.controller.internal.ClusterResourceProvider.SECURITY;
import static org.apache.ambari.server.controller.internal.ProvisionClusterRequest.CONFIG_RECOMMENDATION_STRATEGY;
import static org.apache.ambari.server.controller.internal.ServiceResourceProvider.OPERATION_TYPE;
import static org.apache.ambari.server.topology.Configurable.CONFIGURATIONS;
@@ -31,19 +34,24 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
+import java.util.function.Function;
import org.apache.ambari.annotations.ApiIgnore;
import org.apache.ambari.server.controller.internal.ProvisionAction;
import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
import org.apache.ambari.server.topology.ConfigurableHelper;
import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.Credential;
+import org.apache.ambari.server.topology.SecurityConfiguration;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.swagger.annotations.ApiModel;
@@ -73,30 +81,42 @@ public final class AddServiceRequest {
private final String stackVersion;
private final Set<Service> services;
private final Set<Component> components;
+ private final SecurityConfiguration security;
+ private final Map<String, Credential> credentials;
private final Configuration configuration;
@JsonCreator
- public AddServiceRequest(@JsonProperty(OPERATION_TYPE) OperationType operationType,
- @JsonProperty(CONFIG_RECOMMENDATION_STRATEGY) ConfigRecommendationStrategy recommendationStrategy,
- @JsonProperty(PROVISION_ACTION_PROPERTY)ProvisionAction provisionAction,
- @JsonProperty(STACK_NAME) String stackName,
- @JsonProperty(STACK_VERSION) String stackVersion,
- @JsonProperty(SERVICES) Set<Service> services,
- @JsonProperty(COMPONENTS)Set<Component> components,
- @JsonProperty(CONFIGURATIONS) Collection<? extends Map<String, ?>> configs) {
+ public AddServiceRequest(
+ @JsonProperty(OPERATION_TYPE) OperationType operationType,
+ @JsonProperty(CONFIG_RECOMMENDATION_STRATEGY) ConfigRecommendationStrategy recommendationStrategy,
+ @JsonProperty(PROVISION_ACTION_PROPERTY) ProvisionAction provisionAction,
+ @JsonProperty(STACK_NAME) String stackName,
+ @JsonProperty(STACK_VERSION) String stackVersion,
+ @JsonProperty(SERVICES) Set<Service> services,
+ @JsonProperty(COMPONENTS) Set<Component> components,
+ @JsonProperty(SECURITY) SecurityConfiguration security,
+ @JsonProperty(CREDENTIALS) Set<Credential> credentials,
+ @JsonProperty(CONFIGURATIONS) Collection<? extends Map<String, ?>> configs
+ ) {
this(operationType, recommendationStrategy, provisionAction, stackName, stackVersion, services, components,
- ConfigurableHelper.parseConfigs(configs));
+ security, credentials,
+ ConfigurableHelper.parseConfigs(configs)
+ );
}
- private AddServiceRequest(OperationType operationType,
- ConfigRecommendationStrategy recommendationStrategy,
- ProvisionAction provisionAction,
- String stackName,
- String stackVersion,
- Set<Service> services,
- Set<Component> components,
- Configuration configuration) {
+ private AddServiceRequest(
+ OperationType operationType,
+ ConfigRecommendationStrategy recommendationStrategy,
+ ProvisionAction provisionAction,
+ String stackName,
+ String stackVersion,
+ Set<Service> services,
+ Set<Component> components,
+ SecurityConfiguration security,
+ Set<Credential> credentials,
+ Configuration configuration
+ ) {
this.operationType = null != operationType ? operationType : OperationType.ADD_SERVICE;
this.recommendationStrategy = null != recommendationStrategy ? recommendationStrategy : ConfigRecommendationStrategy.NEVER_APPLY;
this.provisionAction = null != provisionAction ? provisionAction : ProvisionAction.INSTALL_AND_START;
@@ -104,7 +124,11 @@ public final class AddServiceRequest {
this.stackVersion = stackVersion;
this.services = null != services ? services : emptySet();
this.components = null != components ? components : emptySet();
+ this.security = security;
this.configuration = null != configuration ? configuration : new Configuration(new HashMap<>(), new HashMap<>());
+ this.credentials = null != credentials
+ ? credentials.stream().collect(toMap(Credential::getAlias, Function.identity()))
+ : ImmutableMap.of();
checkArgument(!this.services.isEmpty() || !this.components.isEmpty(), "Either services or components must be specified");
}
@@ -172,20 +196,39 @@ public final class AddServiceRequest {
return ConfigurableHelper.convertConfigToMap(configuration);
}
+ @JsonIgnore // TODO confirm: credentials shouldn't be serialized
+ @ApiIgnore
+ public Map<String, Credential> getCredentials() {
+ return credentials;
+ }
+
+ @JsonIgnore
+ @ApiIgnore
+ public Optional<SecurityConfiguration> getSecurity() {
+ return Optional.ofNullable(security);
+ }
+
+ @JsonProperty(SECURITY)
+ @ApiModelProperty(name = SECURITY)
+ public SecurityConfiguration _getSecurity() {
+ return security;
+ }
+
+
// ------- inner classes -------
public enum OperationType {
ADD_SERVICE, DELETE_SERVICE, MOVE_SERVICE
}
- public static class Component {
+ public static final class Component {
static final String COMPONENT_NAME = "component_name";
static final String FQDN = "fqdn";
private String name;
private String fqdn;
- public static final Component of(String name, String fqdn) {
+ public static Component of(String name, String fqdn) {
Component component = new Component();
component.setName(name);
component.setFqdn(fqdn);
@@ -230,12 +273,12 @@ public final class AddServiceRequest {
}
@ApiModel
- public static class Service {
+ public static final class Service {
static final String NAME = "name";
private String name;
- public static final Service of(String name) {
+ public static Service of(String name) {
Service service = new Service();
service.setName(name);
return service;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index 3c4d6b2..0f41ea2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -155,6 +155,11 @@ public interface KerberosHelper {
String ALLOW_RETRY = "allow_retry_on_failure";
/**
+ * Used as key in config maps to store component to host assignment mapping.
+ */
+ String CLUSTER_HOST_INFO = "clusterHostInfo";
+
+ /**
* Toggles Kerberos security to enable it or remove it depending on the state of the cluster.
* <p/>
* The cluster "security_type" property is used to determine the security state of the cluster.
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 23a0d96..120fd7c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -604,12 +604,8 @@ public class KerberosHelperImpl implements KerberosHelper {
}
}
- // Get (and created if needed) the clusterHostInfo map
- Map<String, String> clusterHostInfoMap = configurations.get("clusterHostInfo");
- if (clusterHostInfoMap == null) {
- clusterHostInfoMap = new HashMap<>();
- configurations.put("clusterHostInfo", clusterHostInfoMap);
- }
+ // Get (and create if needed) the clusterHostInfo map
+ Map<String, String> clusterHostInfoMap = configurations.computeIfAbsent(CLUSTER_HOST_INFO, __ -> new HashMap<>());
// Iterate through the recommendations to find the recommended host assignments
for (RecommendationResponse.HostGroup hostGroup : hostGroups) {
@@ -2954,7 +2950,7 @@ public class KerberosHelperImpl implements KerberosHelper {
generalProperties.put("short_date", new SimpleDateFormat("MMddyy").format(new Date()));
// add clusterHostInfo config
- if (configurations.get("clusterHostInfo") == null) {
+ if (configurations.get(CLUSTER_HOST_INFO) == null) {
Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
if (clusterHostInfo != null) {
@@ -2966,7 +2962,7 @@ public class KerberosHelperImpl implements KerberosHelper {
componentHosts.put(entry.getKey(), StringUtils.join(entry.getValue(), ","));
}
- configurations.put("clusterHostInfo", componentHosts);
+ configurations.put(CLUSTER_HOST_INFO, componentHosts);
}
}
configurations.put("principals", principalNames(cluster, configurations));
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index 4eeeea2..18ed30a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -100,8 +100,8 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
public static final String CLUSTER_STATE_PROPERTY_ID = PropertyHelper.getPropertyId("Clusters","state");
static final String BLUEPRINT = "blueprint";
- private static final String SECURITY = "security";
- static final String CREDENTIALS = "credentials";
+ public static final String SECURITY = "security";
+ public static final String CREDENTIALS = "credentials";
private static final String QUICKLINKS_PROFILE = "quicklinks_profile";
private static final String SESSION_ATTRIBUTES = "session_attributes";
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
index 9fe4ee2..ca804a6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
@@ -119,6 +119,11 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
*/
public static final String QUICKLINKS_PROFILE_SERVICES_PROPERTY = "quicklinks_profile/services";
+ public static final String ALIAS = "alias";
+ public static final String PRINCIPAL = "principal";
+ public static final String KEY = "key";
+ public static final String TYPE = "type";
+
/**
* configuration factory
@@ -217,19 +222,19 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
Set<Map<String, String>> credentialsSet = (Set<Map<String, String>>) properties.get(ClusterResourceProvider.CREDENTIALS);
if (credentialsSet != null) {
for (Map<String, String> credentialMap : credentialsSet) {
- String alias = Strings.emptyToNull(credentialMap.get("alias"));
+ String alias = Strings.emptyToNull(credentialMap.get(ALIAS));
if (alias == null) {
throw new InvalidTopologyTemplateException("credential.alias property is missing.");
}
- String principal = Strings.emptyToNull(credentialMap.get("principal"));
+ String principal = Strings.emptyToNull(credentialMap.get(PRINCIPAL));
if (principal == null) {
throw new InvalidTopologyTemplateException("credential.principal property is missing.");
}
- String key = Strings.emptyToNull(credentialMap.get("key"));
+ String key = Strings.emptyToNull(credentialMap.get(KEY));
if (key == null) {
throw new InvalidTopologyTemplateException("credential.key is missing.");
}
- String typeString = Strings.emptyToNull(credentialMap.get("type"));
+ String typeString = Strings.emptyToNull(credentialMap.get(TYPE));
if (typeString == null) {
throw new InvalidTopologyTemplateException("credential.type is missing.");
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
index 382e98b..48b77af 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
@@ -200,6 +200,8 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider
PROPERTY_IDS.add(LDAP_INTEGRATION_DESIRED_PROPERTY_ID);
PROPERTY_IDS.add(OPERATION_TYPE);
+ PROPERTY_IDS.add(ClusterResourceProvider.SECURITY);
+ PROPERTY_IDS.add(ClusterResourceProvider.CREDENTIALS);
// keys
KEY_PROPERTY_IDS.put(Resource.Type.Service, SERVICE_SERVICE_NAME_PROPERTY_ID);
@@ -1234,7 +1236,7 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider
}
private RequestStatusResponse processAddServiceRequest(Map<String, Object> requestProperties, Map<String, String> requestInfoProperties) throws NoSuchParentResourceException {
- AddServiceRequest request = createAddServiceRequest(requestProperties, requestInfoProperties);
+ AddServiceRequest request = createAddServiceRequest(requestInfoProperties);
String clusterName = String.valueOf(requestProperties.get(SERVICE_CLUSTER_NAME_PROPERTY_ID));
try {
return addServiceOrchestrator.processAddServiceRequest(getManagementController().getClusters().getCluster(clusterName), request);
@@ -1243,7 +1245,7 @@ public class ServiceResourceProvider extends AbstractControllerResourceProvider
}
}
- private static AddServiceRequest createAddServiceRequest(Map<String, Object> requestProperties, Map<String, String> requestInfoProperties) {
+ private static AddServiceRequest createAddServiceRequest(Map<String, String> requestInfoProperties) {
return AddServiceRequest.of(requestInfoProperties.get(Request.REQUEST_INFO_BODY_PROPERTY));
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 985c290..cdaeae4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -33,19 +33,22 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor;
import org.apache.ambari.server.controller.ClusterRequest;
import org.apache.ambari.server.controller.ConfigurationRequest;
+import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableSet;
+
/**
* Responsible for cluster configuration.
*/
@@ -57,7 +60,6 @@ public class ClusterConfigurationRequest {
* a regular expression Pattern used to find "clusterHostInfo.(component_name)_host" placeholders in strings
*/
private static final Pattern CLUSTER_HOST_INFO_PATTERN_VARIABLE = Pattern.compile("\\$\\{clusterHostInfo/?([\\w\\-\\.]+)_host(?:\\s*\\|\\s*(.+?))?\\}");
- public static final String CLUSTER_HOST_INFO = "clusterHostInfo";
private AmbariContext ambariContext;
private ClusterTopology clusterTopology;
@@ -159,70 +161,41 @@ public class ClusterConfigurationRequest {
}
private Set<String> configureKerberos(Configuration clusterConfiguration, Map<String, Map<String, String>> existingConfigurations) throws AmbariException {
- Set<String> updatedConfigTypes = new HashSet<>();
-
Cluster cluster = getCluster();
Blueprint blueprint = clusterTopology.getBlueprint();
-
- Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
- Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
+ Set<String> services = ImmutableSet.copyOf(blueprint.getServices());
+ Configuration stackDefaults = blueprint.getStack().getConfiguration(services);
// add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of
// StageUtils.getClusterInfo()
Map<String, String> componentHostsMap = createComponentHostMap(blueprint);
- existingConfigurations.put("clusterHostInfo", componentHostsMap);
+ existingConfigurations.put(KerberosHelper.CLUSTER_HOST_INFO, componentHostsMap);
try {
+ KerberosHelper kerberosHelper = AmbariContext.getController().getKerberosHelper();
+
// generate principals & keytabs for headless identities
- AmbariContext.getController().getKerberosHelper()
- .ensureHeadlessIdentities(cluster, existingConfigurations,
- new HashSet<>(blueprint.getServices()));
+ kerberosHelper.ensureHeadlessIdentities(cluster, existingConfigurations, services);
- // apply Kerberos specific configurations
- Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
- .getServiceConfigurationUpdates(cluster, existingConfigurations,
- createServiceComponentMap(blueprint), null, null, true, false);
+ // get Kerberos specific configurations
+ Map<String, Map<String, String>> updatedConfigs = kerberosHelper.getServiceConfigurationUpdates(
+ cluster, existingConfigurations, createServiceComponentMap(blueprint), null, null, true, false);
- // ******************************************************************************************
// Since Kerberos is being enabled, make sure the cluster-env/security_enabled property is
// set to "true"
- Map<String, String> clusterEnv = updatedConfigs.get("cluster-env");
-
- if(clusterEnv == null) {
- clusterEnv = new HashMap<>();
- updatedConfigs.put("cluster-env", clusterEnv);
- }
-
- clusterEnv.put("security_enabled", "true");
- // ******************************************************************************************
+ updatedConfigs
+ .computeIfAbsent(ConfigHelper.CLUSTER_ENV, __ -> new HashMap<>())
+ .put("security_enabled", "true");
- for (String configType : updatedConfigs.keySet()) {
- // apply only if config type has related services in Blueprint
- if (blueprint.isValidConfigType(configType)) {
- Map<String, String> propertyMap = updatedConfigs.get(configType);
- Map<String, String> clusterConfigProperties = existingConfigurations.get(configType);
- Map<String, String> stackDefaultConfigProperties = stackDefaultProps.get(configType);
- for (String property : propertyMap.keySet()) {
- // update value only if property value configured in Blueprint / ClusterTemplate is not a custom one
- String currentValue = clusterConfiguration.getPropertyValue(configType, property);
- String newValue = propertyMap.get(property);
- if (!propertyHasCustomValue(clusterConfigProperties, stackDefaultConfigProperties, property) &&
- (currentValue == null || !currentValue.equals(newValue))) {
-
- LOG.debug("Update Kerberos related config property: {} {} {}", configType, property, propertyMap.get
- (property));
- clusterConfiguration.setProperty(configType, property, newValue);
- updatedConfigTypes.add(configType);
- }
- }
- }
- }
+ updatedConfigs.keySet().removeIf(configType -> !blueprint.isValidConfigType(configType));
+ // apply Kerberos specific configurations
+ return clusterConfiguration.applyUpdatesToStackDefaultProperties(stackDefaults, existingConfigurations, updatedConfigs);
} catch (KerberosInvalidConfigurationException e) {
LOG.error("An exception occurred while doing Kerberos related configuration update: " + e, e);
}
- return updatedConfigTypes;
+ return ImmutableSet.of();
}
/**
@@ -238,58 +211,20 @@ public class ClusterConfigurationRequest {
if(services != null) {
for (String service : services) {
Collection<String> components = blueprint.getComponents(service);
- serviceComponents.put(service,
- (components == null)
- ? Collections.emptySet()
- : new HashSet<>(blueprint.getComponents(service)));
+ Set<String> componentSet = components == null ? ImmutableSet.of() : ImmutableSet.copyOf(components);
+ serviceComponents.put(service, componentSet);
}
}
return serviceComponents;
}
- /**
- * Returns true if the property exists in clusterConfigProperties and has a custom user defined value. Property has
- * custom value in case we there's no stack default value for it or it's not equal to stack default value.
- * @param clusterConfigProperties
- * @param stackDefaultConfigProperties
- * @param property
- * @return
- */
- private boolean propertyHasCustomValue(Map<String, String> clusterConfigProperties, Map<String, String>
- stackDefaultConfigProperties, String property) {
-
- boolean propertyHasCustomValue = false;
- if (clusterConfigProperties != null) {
- String propertyValue = clusterConfigProperties.get(property);
- if (propertyValue != null) {
- if (stackDefaultConfigProperties != null) {
- String stackDefaultValue = stackDefaultConfigProperties.get(property);
- if (stackDefaultValue != null) {
- propertyHasCustomValue = !propertyValue.equals(stackDefaultValue);
- } else {
- propertyHasCustomValue = true;
- }
- } else {
- propertyHasCustomValue = true;
- }
- }
- }
- return propertyHasCustomValue;
- }
-
private Map<String, String> createComponentHostMap(Blueprint blueprint) {
- Map<String, String> componentHostsMap = new HashMap<>();
- for (String service : blueprint.getServices()) {
- Collection<String> components = blueprint.getComponents(service);
- for (String component : components) {
- Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component);
- // retrieve corresponding clusterInfoKey for component using StageUtils
- String clusterInfoKey = StageUtils.getClusterHostInfoKey(component);
- componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ","));
- }
- }
- return componentHostsMap;
+ return StageUtils.createComponentHostMap(
+ blueprint.getServices(),
+ blueprint::getComponents,
+ (service, component) -> clusterTopology.getHostAssignmentsForComponent(component)
+ );
}
private Collection<String> getRequiredHostgroupsForKerberosConfiguration() {
@@ -301,7 +236,7 @@ public class ClusterConfigurationRequest {
Configuration clusterConfiguration = clusterTopology.getConfiguration();
Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties();
- existingConfigurations.put(CLUSTER_HOST_INFO, new HashMap<>());
+ existingConfigurations.put(KerberosHelper.CLUSTER_HOST_INFO, new HashMap<>());
// apply Kerberos specific configurations
Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
@@ -398,7 +333,7 @@ public class ClusterConfigurationRequest {
*/
private void setConfigurationsOnCluster(List<BlueprintServiceConfigRequest> configurationRequests,
String tag, Set<String> updatedConfigTypes) {
- String clusterName = null;
+ String clusterName;
try {
clusterName = ambariContext.getClusterName(clusterTopology.getClusterId());
} catch (AmbariException e) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index bbd4502..8363716 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -22,12 +22,19 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Configuration for a topology entity such as a blueprint, hostgroup or cluster.
*/
public class Configuration {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Configuration.class);
+
/**
* properties for this configuration instance
*/
@@ -47,6 +54,64 @@ public class Configuration {
return new Configuration(new HashMap<>(), new HashMap<>());
}
+ /**
+ * Apply configuration changes from {@code updatedConfigs} to {@code config}, but only
+ * change properties that are either absent in the existing config, or have values that
+ * match the stack default config.
+ *
+ * @return config types that had any updates applied
+ */
+ public Set<String> applyUpdatesToStackDefaultProperties(Configuration stackDefaultConfig, Map<String, Map<String, String>> existingConfigurations, Map<String, Map<String, String>> updatedConfigs) {
+ Set<String> updatedConfigTypes = new HashSet<>();
+
+ Map<String, Map<String, String>> stackDefaults = stackDefaultConfig.getProperties();
+
+ for (Map.Entry<String, Map<String, String>> configEntry : updatedConfigs.entrySet()) {
+ String configType = configEntry.getKey();
+ Map<String, String> propertyMap = configEntry.getValue();
+ Map<String, String> clusterConfigProperties = existingConfigurations.get(configType);
+ Map<String, String> stackDefaultConfigProperties = stackDefaults.get(configType);
+ for (Map.Entry<String, String> propertyEntry : propertyMap.entrySet()) {
+ String property = propertyEntry.getKey();
+ String newValue = propertyEntry.getValue();
+ String currentValue = getPropertyValue(configType, property);
+
+ if (!propertyHasCustomValue(clusterConfigProperties, stackDefaultConfigProperties, property) && !Objects.equals(currentValue, newValue)) {
+ LOG.debug("Update config property {}/{}: {} -> {}", configType, property, currentValue, newValue);
+ setProperty(configType, property, newValue);
+ updatedConfigTypes.add(configType);
+ }
+ }
+ }
+
+ return updatedConfigTypes;
+ }
+
+ /**
+ * Returns true if the property exists in clusterConfigProperties and has a custom user defined value. Property has
+ * custom value in case we there's no stack default value for it or it's not equal to stack default value.
+ */
+ private static boolean propertyHasCustomValue(Map<String, String> clusterConfigProperties, Map<String, String> stackDefaultConfigProperties, String property) {
+
+ boolean propertyHasCustomValue = false;
+ if (clusterConfigProperties != null) {
+ String propertyValue = clusterConfigProperties.get(property);
+ if (propertyValue != null) {
+ if (stackDefaultConfigProperties != null) {
+ String stackDefaultValue = stackDefaultConfigProperties.get(property);
+ if (stackDefaultValue != null) {
+ propertyHasCustomValue = !propertyValue.equals(stackDefaultValue);
+ } else {
+ propertyHasCustomValue = true;
+ }
+ } else {
+ propertyHasCustomValue = true;
+ }
+ }
+ }
+ return propertyHasCustomValue;
+ }
+
public Configuration copy() {
Configuration parent = parentConfiguration;
parentConfiguration = null;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Credential.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Credential.java
index 7ced7ca..51f00fb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Credential.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Credential.java
@@ -18,53 +18,108 @@
package org.apache.ambari.server.topology;
+import static org.apache.ambari.server.controller.internal.ProvisionClusterRequest.ALIAS;
+import static org.apache.ambari.server.controller.internal.ProvisionClusterRequest.KEY;
+import static org.apache.ambari.server.controller.internal.ProvisionClusterRequest.PRINCIPAL;
+import static org.apache.ambari.server.controller.internal.ProvisionClusterRequest.TYPE;
+
+import java.util.Objects;
+
import org.apache.ambari.server.security.encryption.CredentialStoreType;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
/**
* Holds credential info submitted in a cluster create template.
*/
+@ApiModel
public class Credential {
/**
* Credential alias like kdc.admin.credential.
*/
- private String alias;
+ private final String alias;
/**
* Name of a principal.
*/
- private String principal;
+ private final String principal;
/**
* Key of credential.
*/
- private String key;
+ private final String key;
/**
* Type of credential store.
*/
- private CredentialStoreType type;
+ private final CredentialStoreType type;
- public Credential(String alias, String principal, String key, CredentialStoreType type) {
+ @JsonCreator
+ public Credential(
+ @JsonProperty(ALIAS) String alias,
+ @JsonProperty(PRINCIPAL) String principal,
+ @JsonProperty(KEY) String key,
+ @JsonProperty(TYPE) CredentialStoreType type
+ ) {
this.alias = alias;
this.principal = principal;
this.key = key;
this.type = type;
}
+ @JsonProperty(ALIAS)
+ @ApiModelProperty(name = ALIAS)
public String getAlias() {
return alias;
}
+ @JsonProperty(PRINCIPAL)
+ @ApiModelProperty(name = PRINCIPAL)
public String getPrincipal() {
return principal;
}
+ @JsonProperty(KEY)
+ @ApiModelProperty(name = KEY)
public String getKey() {
return key;
}
+ @JsonProperty(TYPE)
+ @ApiModelProperty(name = TYPE)
public CredentialStoreType getType() {
return type;
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || obj.getClass() != getClass()) {
+ return false;
+ }
+
+ Credential other = (Credential) obj;
+
+ return Objects.equals(alias, other.alias) &&
+ Objects.equals(principal, other.principal) &&
+ Objects.equals(key, other.key) &&
+ Objects.equals(type, other.type);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(alias, principal, key, type);
+ }
+
+ @Override
+ public String toString() {
+ return alias;
+ }
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/SecurityConfiguration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/SecurityConfiguration.java
index 72542c1..47cd117 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/SecurityConfiguration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/SecurityConfiguration.java
@@ -18,15 +18,32 @@
package org.apache.ambari.server.topology;
+import static org.apache.ambari.server.topology.SecurityConfigurationFactory.KERBEROS_DESCRIPTOR_PROPERTY_ID;
+import static org.apache.ambari.server.topology.SecurityConfigurationFactory.KERBEROS_DESCRIPTOR_REFERENCE_PROPERTY_ID;
+import static org.apache.ambari.server.topology.SecurityConfigurationFactory.TYPE_PROPERTY_ID;
+
+import java.util.Objects;
+
import org.apache.ambari.server.state.SecurityType;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
/**
* Holds security related properties, the securityType and security descriptor (in case of KERBEROS
* kerberos_descriptor) either contains the whole descriptor or just the reference to it.
*
*/
+@ApiModel
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
public class SecurityConfiguration {
+ public static final SecurityConfiguration NONE = new SecurityConfiguration(SecurityType.NONE);
+
/**
* Security Type
*/
@@ -46,21 +63,54 @@ public class SecurityConfiguration {
this.type = type;
}
- public SecurityConfiguration(SecurityType type, String descriptorReference, String descriptor) {
+ @JsonCreator
+ public SecurityConfiguration(
+ @JsonProperty(TYPE_PROPERTY_ID) SecurityType type,
+ @JsonProperty(KERBEROS_DESCRIPTOR_REFERENCE_PROPERTY_ID) String descriptorReference,
+ @JsonProperty(KERBEROS_DESCRIPTOR_PROPERTY_ID) String descriptor
+ ) {
this.type = type;
this.descriptorReference = descriptorReference;
this.descriptor = descriptor;
}
+ @JsonProperty(TYPE_PROPERTY_ID)
+ @ApiModelProperty(name = TYPE_PROPERTY_ID)
public SecurityType getType() {
return type;
}
+ @JsonProperty(KERBEROS_DESCRIPTOR_REFERENCE_PROPERTY_ID)
+ @ApiModelProperty(name = KERBEROS_DESCRIPTOR_REFERENCE_PROPERTY_ID)
public String getDescriptor() {
return descriptor;
}
+ @JsonProperty(KERBEROS_DESCRIPTOR_PROPERTY_ID)
+ @ApiModelProperty(name = KERBEROS_DESCRIPTOR_PROPERTY_ID)
public String getDescriptorReference() {
return descriptorReference;
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || obj.getClass() != getClass()) {
+ return false;
+ }
+
+ SecurityConfiguration other = (SecurityConfiguration) obj;
+
+ return Objects.equals(type, other.type) &&
+ Objects.equals(descriptor, other.descriptor) &&
+ Objects.equals(descriptorReference, other.descriptorReference);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, descriptor, descriptorReference);
+ }
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index 004455d..5c030b6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -48,12 +48,12 @@ import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
import org.apache.ambari.server.controller.internal.BaseClusterRequest;
import org.apache.ambari.server.controller.internal.CalculatedStatus;
-import org.apache.ambari.server.controller.internal.CredentialResourceProvider;
import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
import org.apache.ambari.server.controller.internal.RequestImpl;
import org.apache.ambari.server.controller.internal.ScaleClusterRequest;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
+import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.RequestStatus;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
@@ -76,6 +76,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.host.HostImpl;
import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfile;
+import org.apache.ambari.server.topology.addservice.ResourceProviderAdapter;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTask;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory;
import org.apache.ambari.server.topology.validators.TopologyValidatorService;
@@ -84,6 +85,8 @@ import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Singleton;
import com.google.inject.persist.Transactional;
@@ -395,32 +398,23 @@ public class TopologyManager {
}
}
- private void submitCredential(String clusterName, Credential credential) {
-
- ResourceProvider provider =
- ambariContext.getClusterController().ensureResourceProvider(Resource.Type.Credential);
-
- Map<String, Object> properties = new HashMap<>();
- properties.put(CredentialResourceProvider.CREDENTIAL_CLUSTER_NAME_PROPERTY_ID, clusterName);
- properties.put(CredentialResourceProvider.CREDENTIAL_ALIAS_PROPERTY_ID, KDC_ADMIN_CREDENTIAL);
- properties.put(CredentialResourceProvider.CREDENTIAL_PRINCIPAL_PROPERTY_ID, credential.getPrincipal());
- properties.put(CredentialResourceProvider.CREDENTIAL_KEY_PROPERTY_ID, credential.getKey());
- properties.put(CredentialResourceProvider.CREDENTIAL_TYPE_PROPERTY_ID, credential.getType().name());
-
- org.apache.ambari.server.controller.spi.Request request = new RequestImpl(Collections.emptySet(),
- Collections.singleton(properties), Collections.emptyMap(), null);
-
+ private static void submitCredential(String clusterName, Credential credential) {
+ ResourceProvider provider = AmbariContext.getClusterController().ensureResourceProvider(Resource.Type.Credential);
+ Map<String, Object> credentialProperties = ResourceProviderAdapter.createCredentialRequestProperties(clusterName, credential);
+ Request request = new RequestImpl(ImmutableSet.of(), ImmutableSet.of(credentialProperties), ImmutableMap.of(), null);
+ String baseMessage = String.format("Failed to add credential %s to cluster %s", credential.getAlias(), clusterName);
try {
RequestStatus status = provider.createResources(request);
if (status.getStatus() != RequestStatus.Status.Complete) {
- throw new RuntimeException("Failed to attach kerberos_descriptor artifact to cluster!");
+ String msg = String.format("%s, received status: %s", baseMessage, status.getStatus());
+ LOG.error(msg);
+ throw new RuntimeException(msg);
}
- } catch (SystemException | UnsupportedPropertyException | NoSuchParentResourceException e) {
- throw new RuntimeException("Failed to attach kerberos_descriptor artifact to cluster: " + e);
- } catch (ResourceAlreadyExistsException e) {
- throw new RuntimeException("Failed to attach kerberos_descriptor artifact to cluster as resource already exists.");
+ } catch (ResourceAlreadyExistsException | SystemException | UnsupportedPropertyException | NoSuchParentResourceException e) {
+ String msg = String.format("%s, %s", baseMessage, e);
+ LOG.error(msg);
+ throw new RuntimeException(msg, e);
}
-
}
/**
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
index 8662746..15c3b1f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceInfo.java
@@ -22,6 +22,7 @@ import static java.util.stream.Collectors.joining;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.controller.AddServiceRequest;
import org.apache.ambari.server.controller.internal.RequestStageContainer;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.topology.Configuration;
@@ -31,13 +32,15 @@ import org.apache.ambari.server.topology.Configuration;
*/
public final class AddServiceInfo {
+ private final AddServiceRequest request;
private final String clusterName;
private final Stack stack;
private final Map<String, Map<String, Set<String>>> newServices;
private final RequestStageContainer stages;
private final Configuration config;
- public AddServiceInfo(String clusterName, Stack stack, Configuration config, RequestStageContainer stages, Map<String, Map<String, Set<String>>> newServices) {
+ public AddServiceInfo(AddServiceRequest request, String clusterName, Stack stack, Configuration config, RequestStageContainer stages, Map<String, Map<String, Set<String>>> newServices) {
+ this.request = request;
this.clusterName = clusterName;
this.stack = stack;
this.newServices = newServices;
@@ -50,6 +53,10 @@ public final class AddServiceInfo {
return "AddServiceRequest(" + stages.getId() + ")";
}
+ public AddServiceRequest getRequest() {
+ return request;
+ }
+
public String clusterName() {
return clusterName;
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
index d30ab09..a669b94 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/AddServiceOrchestrator.java
@@ -17,6 +17,8 @@
*/
package org.apache.ambari.server.topology.addservice;
+import static com.google.common.base.Preconditions.checkArgument;
+
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -31,16 +33,25 @@ import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.controller.AddServiceRequest;
import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.internal.RequestStageContainer;
import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.SecurityType;
+import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.utils.StageUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
@Singleton
public class AddServiceOrchestrator {
@@ -58,6 +69,9 @@ public class AddServiceOrchestrator {
@Inject
private RequestFactory requestFactory;
+ @Inject
+ private ConfigHelper configHelper;
+
public RequestStatusResponse processAddServiceRequest(Cluster cluster, AddServiceRequest request) {
LOG.info("Received {} request for {}: {}", request.getOperationType(), cluster.getClusterName(), request);
@@ -80,6 +94,13 @@ public class AddServiceOrchestrator {
private AddServiceInfo validate(Cluster cluster, AddServiceRequest request) {
LOG.info("Validating {}", request);
+ request.getSecurity().ifPresent(requestSecurity ->
+ checkArgument(requestSecurity.getType() == cluster.getSecurityType(),
+ "Security type in the request (%s), if specified, should match cluster's security type (%s)",
+ requestSecurity.getType(), cluster.getSecurityType()
+ )
+ );
+
Map<String, Map<String, Set<String>>> newServices = new LinkedHashMap<>();
StackId stackId = new StackId(request.getStackName(), request.getStackVersion());
@@ -114,10 +135,12 @@ public class AddServiceOrchestrator {
}
Configuration config = request.getConfiguration();
- config.setParentConfiguration(stack.getValidDefaultConfig());
+ Configuration clusterConfig = getClusterDesiredConfigs(cluster);
+ clusterConfig.setParentConfiguration(stack.getValidDefaultConfig());
+ config.setParentConfiguration(clusterConfig);
RequestStageContainer stages = new RequestStageContainer(actionManager.getNextRequestId(), null, requestFactory, actionManager);
- AddServiceInfo validatedRequest = new AddServiceInfo(cluster.getClusterName(), stack, config, stages, newServices);
+ AddServiceInfo validatedRequest = new AddServiceInfo(request, cluster.getClusterName(), stack, config, stages, newServices);
stages.setRequestContext(validatedRequest.describe());
return validatedRequest;
}
@@ -149,13 +172,47 @@ public class AddServiceOrchestrator {
*/
private void createResources(AddServiceInfo request) {
LOG.info("Creating resources for {}", request);
- resourceProviders.updateExistingConfigs(request);
+
+ Cluster cluster = getCluster(request.clusterName());
+ Set<String> existingServices = cluster.getServices().keySet();
+
+ resourceProviders.createCredentials(request);
+
resourceProviders.createServices(request);
resourceProviders.createComponents(request);
- resourceProviders.createConfigs(request);
+
resourceProviders.updateServiceDesiredState(request, State.INSTALLED);
resourceProviders.updateServiceDesiredState(request, State.STARTED);
resourceProviders.createHostComponents(request);
+
+ configureKerberos(request, cluster, existingServices);
+ resourceProviders.updateExistingConfigs(request, existingServices);
+ resourceProviders.createConfigs(request);
+ }
+
+ private void configureKerberos(AddServiceInfo request, Cluster cluster, Set<String> existingServices) {
+ if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+ LOG.info("Configuring Kerberos for {}", request);
+
+ Configuration stackDefaultConfig = request.getStack().getValidDefaultConfig();
+ Set<String> newServices = request.newServices().keySet();
+ Set<String> services = ImmutableSet.copyOf(Sets.union(newServices, existingServices));
+ Map<String, Map<String, String>> existingConfigurations = request.getConfig().getFullProperties();
+ existingConfigurations.put(KerberosHelper.CLUSTER_HOST_INFO, createComponentHostMap(cluster));
+
+ try {
+ KerberosHelper kerberosHelper = controller.getKerberosHelper();
+ kerberosHelper.ensureHeadlessIdentities(cluster, existingConfigurations, services);
+ request.getConfig().applyUpdatesToStackDefaultProperties(stackDefaultConfig, existingConfigurations,
+ kerberosHelper.getServiceConfigurationUpdates(
+ cluster, existingConfigurations, createServiceComponentMap(cluster), null, existingServices, true, true
+ )
+ );
+ } catch (AmbariException | KerberosInvalidConfigurationException e) {
+ LOG.error("Error configuring Kerberos: {}", e, e);
+ throw new RuntimeException(e);
+ }
+ }
}
private void createHostTasks(AddServiceInfo request) {
@@ -172,4 +229,67 @@ public class AddServiceOrchestrator {
}
}
+ private static Map<String, String> createComponentHostMap(Cluster cluster) {
+ return StageUtils.createComponentHostMap(
+ cluster.getServices().keySet(),
+ service -> getComponentsForService(cluster, service),
+ (service, component) -> getHostsForServiceComponent(cluster, service, component)
+ );
+ }
+
+ private static Set<String> getHostsForServiceComponent(Cluster cluster, String service, String component) {
+ try {
+ return cluster.getService(service).getServiceComponent(component).getServiceComponentsHosts();
+ } catch (AmbariException e) {
+ LOG.error("Error getting components of service {}: {}", service, e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Set<String> getComponentsForService(Cluster cluster, String service) {
+ try {
+ return cluster.getService(service).getServiceComponents().keySet();
+ } catch (AmbariException e) {
+ LOG.error("Error getting components of service {}: {}", service, e, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static Map<String, Set<String>> createServiceComponentMap(Cluster cluster) {
+ Map<String, Set<String>> serviceComponentMap = new HashMap<>();
+ for (Map.Entry<String, Service> e : cluster.getServices().entrySet()) {
+ serviceComponentMap.put(e.getKey(), ImmutableSet.copyOf(e.getValue().getServiceComponents().keySet()));
+ }
+ return serviceComponentMap;
+ }
+
+ private Configuration getClusterDesiredConfigs(Cluster cluster) {
+ Map<String, Map<String, String>> desiredConfigTags = getDesiredTags(cluster);
+
+ return new Configuration(
+ configHelper.getEffectiveConfigProperties(cluster, desiredConfigTags),
+ configHelper.getEffectiveConfigAttributes(cluster, desiredConfigTags)
+ );
+ }
+
+ private Map<String, Map<String, String>> getDesiredTags(Cluster cluster) {
+ try {
+ return configHelper.getEffectiveDesiredTags(cluster, null);
+ } catch (AmbariException e) {
+ String msg = String.format("Error getting tags for desired config of cluster %s", cluster.getClusterName());
+ LOG.error(msg);
+ throw new IllegalStateException(msg, e);
+ }
+ }
+
+ private Cluster getCluster(String clusterName) {
+ try {
+ return controller.getClusters().getCluster(clusterName);
+ } catch (AmbariException e) {
+ String msg = String.format("Cannot find cluster %s", clusterName);
+ LOG.error(msg);
+ throw new IllegalStateException(msg, e);
+ }
+ }
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/ResourceProviderAdapter.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/ResourceProviderAdapter.java
index ee52c7f..db8a57a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/ResourceProviderAdapter.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/addservice/ResourceProviderAdapter.java
@@ -36,6 +36,7 @@ import org.apache.ambari.server.controller.ClusterRequest;
import org.apache.ambari.server.controller.ConfigurationRequest;
import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
+import org.apache.ambari.server.controller.internal.CredentialResourceProvider;
import org.apache.ambari.server.controller.internal.HostComponentResourceProvider;
import org.apache.ambari.server.controller.internal.RequestImpl;
import org.apache.ambari.server.controller.internal.RequestOperationLevel;
@@ -56,10 +57,9 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.security.authorization.AuthorizationException;
-import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.State;
-import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.Credential;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,9 +79,6 @@ public class ResourceProviderAdapter {
@Inject
private AmbariManagementController controller;
- @Inject
- private ConfigHelper configHelper;
-
public void createServices(AddServiceInfo request) {
LOG.info("Creating service resources for {}", request);
@@ -89,7 +86,7 @@ public class ResourceProviderAdapter {
.map(service -> createServiceRequestProperties(request, service))
.collect(toSet());
- createResources(request, properties, Resource.Type.Service);
+ createResources(request, properties, Resource.Type.Service, false);
}
public void createComponents(AddServiceInfo request) {
@@ -100,7 +97,7 @@ public class ResourceProviderAdapter {
.map(component -> createComponentRequestProperties(request, componentsOfService.getKey(), component)))
.collect(toSet());
- createResources(request, properties, Resource.Type.Component);
+ createResources(request, properties, Resource.Type.Component, false);
}
public void createHostComponents(AddServiceInfo request) {
@@ -112,7 +109,7 @@ public class ResourceProviderAdapter {
.map(host -> createHostComponentRequestProperties(request, componentsOfService.getKey(), hostsOfComponent.getKey(), host))))
.collect(toSet());
- createResources(request, properties, Resource.Type.HostComponent);
+ createResources(request, properties, Resource.Type.HostComponent, false);
}
public void createConfigs(AddServiceInfo request) {
@@ -121,9 +118,22 @@ public class ResourceProviderAdapter {
updateCluster(request, requests, "Error creating configurations for %s");
}
- public void updateExistingConfigs(AddServiceInfo request) {
+ public void createCredentials(AddServiceInfo request) {
+ if (!request.getRequest().getCredentials().isEmpty()) {
+ LOG.info("Creating {} credential(s) for {}", request.getRequest().getCredentials().size(), request);
+
+ request.getRequest().getCredentials().values().stream()
+ .peek(credential -> LOG.debug("Creating credential {}", credential))
+ .map(credential -> createCredentialRequestProperties(request.clusterName(), credential))
+ .forEach(
+ properties -> createResources(request, ImmutableSet.of(properties), Resource.Type.Credential, true)
+ );
+ }
+ }
+
+ public void updateExistingConfigs(AddServiceInfo request, Set<String> existingServices) {
LOG.info("Updating existing configurations for {}", request);
- Set<ClusterRequest> requests = createConfigRequestsForExistingServices(request);
+ Set<ClusterRequest> requests = createConfigRequestsForExistingServices(request, existingServices);
updateCluster(request, requests, "Error updating configurations for %s");
}
@@ -154,15 +164,19 @@ public class ResourceProviderAdapter {
}
}
- private static void createResources(AddServiceInfo request, Set<Map<String, Object>> properties, Resource.Type resourceType) {
+ private static void createResources(AddServiceInfo request, Set<Map<String, Object>> properties, Resource.Type resourceType, boolean okIfExists) {
Request internalRequest = new RequestImpl(null, properties, null, null);
ResourceProvider rp = getClusterController().ensureResourceProvider(resourceType);
try {
rp.createResources(internalRequest);
} catch (UnsupportedPropertyException | SystemException | ResourceAlreadyExistsException | NoSuchParentResourceException e) {
- String msg = String.format("Error creating resources %s for %s", resourceType, request);
- LOG.error(msg, e);
- throw new RuntimeException(msg, e);
+ if (okIfExists && e instanceof ResourceAlreadyExistsException) {
+ LOG.info("Resource already exists: {}, no need to create", e.getMessage());
+ } else {
+ String msg = String.format("Error creating resources %s for %s", resourceType, request);
+ LOG.error(msg, e);
+ throw new RuntimeException(msg, e);
+ }
}
}
@@ -229,6 +243,18 @@ public class ResourceProviderAdapter {
return properties.build();
}
+ public static Map<String, Object> createCredentialRequestProperties(String clusterName, Credential credential) {
+ ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
+
+ properties.put(CredentialResourceProvider.CREDENTIAL_CLUSTER_NAME_PROPERTY_ID, clusterName);
+ properties.put(CredentialResourceProvider.CREDENTIAL_ALIAS_PROPERTY_ID, credential.getAlias());
+ properties.put(CredentialResourceProvider.CREDENTIAL_PRINCIPAL_PROPERTY_ID, credential.getPrincipal());
+ properties.put(CredentialResourceProvider.CREDENTIAL_KEY_PROPERTY_ID, credential.getKey());
+ properties.put(CredentialResourceProvider.CREDENTIAL_TYPE_PROPERTY_ID, credential.getType().name());
+
+ return properties.build();
+ }
+
private static Set<ClusterRequest> createConfigRequestsForNewServices(AddServiceInfo request) {
Map<String, Map<String, String>> fullProperties = request.getConfig().getFullProperties();
Map<String, Map<String, Map<String, String>>> fullAttributes = request.getConfig().getFullAttributes();
@@ -240,17 +266,7 @@ public class ResourceProviderAdapter {
);
}
- private Set<ClusterRequest> createConfigRequestsForExistingServices(AddServiceInfo request) {
- Cluster cluster = getCluster(request.clusterName());
- Map<String, Map<String, String>> desiredConfigTags = getDesiredTags(cluster);
- Configuration mergedConfig = new Configuration(
- request.getConfig().getProperties(), request.getConfig().getAttributes(),
- new Configuration(
- configHelper.getEffectiveConfigProperties(cluster, desiredConfigTags),
- configHelper.getEffectiveConfigAttributes(cluster, desiredConfigTags)
- )
- );
-
+ private Set<ClusterRequest> createConfigRequestsForExistingServices(AddServiceInfo request, Set<String> existingServices) {
Set<String> configTypesInRequest = ImmutableSet.copyOf(
Sets.difference(
Sets.union(
@@ -259,11 +275,11 @@ public class ResourceProviderAdapter {
ImmutableSet.of(ConfigHelper.CLUSTER_ENV))
);
- Map<String, Map<String, String>> fullProperties = mergedConfig.getFullProperties();
- Map<String, Map<String, Map<String, String>>> fullAttributes = mergedConfig.getFullAttributes();
+ Map<String, Map<String, String>> fullProperties = request.getConfig().getFullProperties();
+ Map<String, Map<String, Map<String, String>>> fullAttributes = request.getConfig().getFullAttributes();
Set<ClusterRequest> clusterRequests = createConfigRequestsForServices(
- cluster.getServices().keySet(),
+ existingServices,
configTypesInRequest::contains,
request, fullProperties, fullAttributes
);
@@ -323,26 +339,6 @@ public class ResourceProviderAdapter {
return Optional.of(clusterRequest);
}
- private Map<String, Map<String, String>> getDesiredTags(Cluster cluster) {
- try {
- return configHelper.getEffectiveDesiredTags(cluster, null);
- } catch (AmbariException e) {
- String msg = String.format("Error getting tags for desired config of cluster %s", cluster.getClusterName());
- LOG.error(msg);
- throw new IllegalStateException(msg, e);
- }
- }
-
- private Cluster getCluster(String clusterName) {
- try {
- return controller.getClusters().getCluster(clusterName);
- } catch (AmbariException e) {
- String msg = String.format("Cannot find cluster %s", clusterName);
- LOG.error(msg);
- throw new IllegalStateException(msg, e);
- }
- }
-
private static Predicate predicateForNewServices(AddServiceInfo request, String category) {
return new AndPredicate(
new EqualsPredicate<>(PropertyHelper.getPropertyId(category, ClusterResourceProvider.CLUSTER_NAME), request.clusterName()),
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
index f6967a0..82f3306 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -443,6 +444,8 @@ public class ShellCommandUtil {
env.putAll(vars);
}
+ LOG.debug("Executing the command {}", Arrays.toString(processArgs));
+
Process process;
if (WINDOWS) {
synchronized (WindowsProcessLaunchLock) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index 6ce01a9..d0351d0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -45,6 +45,8 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.function.BiFunction;
+import java.util.function.Function;
import javax.xml.bind.JAXBException;
@@ -617,4 +619,26 @@ public class StageUtils {
}
}
}
+
+ /**
+ * Create a component -> hosts mapping that can be used for clusterHostInfo.
+ * Component names are transformed to clusterHostInfo keys ("_hosts" is appended).
+ * List of hosts is comma-separated.
+ *
+ * @param services collection of services to create the mapping for
+ * @param componentsLookup function to find components of a given service
+ * @param hostAssignmentLookup function to find hosts of a given (service, component) pair
+ * @return component names
+ */
+ public static Map<String, String> createComponentHostMap(Collection<String> services, Function<String, Collection<String>> componentsLookup, BiFunction<String, String, Collection<String>> hostAssignmentLookup) {
+ Map<String, String> componentHostsMap = new HashMap<>();
+ for (String service : services) {
+ Collection<String> components = componentsLookup.apply(service);
+ for (String component : components) {
+ Collection<String> hosts = hostAssignmentLookup.apply(service, component);
+ componentHostsMap.put(getClusterHostInfoKey(component), StringUtils.join(hosts, ","));
+ }
+ }
+ return componentHostsMap;
+ }
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
index ed1a865..96c3d75 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AddServiceRequestTest.java
@@ -42,11 +42,16 @@ import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import org.apache.ambari.server.controller.internal.ProvisionAction;
+import org.apache.ambari.server.security.encryption.CredentialStoreType;
+import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
import org.apache.ambari.server.topology.Configurable;
import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.Credential;
+import org.apache.ambari.server.topology.SecurityConfiguration;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -110,6 +115,16 @@ public class AddServiceRequestTest {
ImmutableSet.of(Service.of("STORM"), Service.of("BEACON")),
request.getServices());
+ assertEquals(
+ Optional.of(new SecurityConfiguration(SecurityType.KERBEROS, "ref_to_kerb_desc", null)),
+ request.getSecurity());
+
+ assertEquals(
+ ImmutableMap.of(
+ "kdc.admin.credential", new Credential( "kdc.admin.credential", "admin/admin@EXAMPLE.COM", "k", CredentialStoreType.TEMPORARY)
+ ),
+ request.getCredentials()
+ );
}
@Test
@@ -131,6 +146,8 @@ public class AddServiceRequestTest {
assertEquals(INSTALL_AND_START, request.getProvisionAction());
assertNull(request.getStackName());
assertNull(request.getStackVersion());
+ assertEquals(Optional.empty(), request.getSecurity());
+ assertEquals(ImmutableMap.of(), request.getCredentials());
Configuration configuration = request.getConfiguration();
assertTrue(configuration.getFullAttributes().isEmpty());
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index 771b89f..5aa8ce6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -230,7 +230,7 @@ public class ClusterConfigurationRequestTest {
expectLastCall().andReturn(controller).anyTimes();
expect(controller.getClusters()).andReturn(clusters).anyTimes();
- expect(controller.getKerberosHelper()).andReturn(kerberosHelper).times(2);
+ expect(controller.getKerberosHelper()).andReturn(kerberosHelper).times(1);
expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes();
diff --git a/ambari-server/src/test/resources/add_service_api/request1.json b/ambari-server/src/test/resources/add_service_api/request1.json
index cf4ce62..dbd12b6 100644
--- a/ambari-server/src/test/resources/add_service_api/request1.json
+++ b/ambari-server/src/test/resources/add_service_api/request1.json
@@ -21,6 +21,20 @@
}
],
+ "security": {
+ "type": "KERBEROS",
+ "kerberos_descriptor_reference": "ref_to_kerb_desc"
+ },
+
+ "credentials": [
+ {
+ "alias": "kdc.admin.credential",
+ "principal": "admin/admin@EXAMPLE.COM",
+ "key": "k",
+ "type": "TEMPORARY"
+ }
+ ],
+
"configurations" : [
{
"storm-site" : {
@@ -36,4 +50,4 @@
}
]
-}
\ No newline at end of file
+}