You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by js...@apache.org on 2015/04/27 07:53:02 UTC
[08/13] ambari git commit: AMBARI-10750. Initial merge of advanced
api provisioning work.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
new file mode 100644
index 0000000..70d1907
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.DependencyInfo;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+/**
+ * Default blueprint validator.
+ */
+public class BlueprintValidatorImpl implements BlueprintValidator {
+
+ private final Blueprint blueprint;
+ private final Stack stack;
+
+ public BlueprintValidatorImpl(Blueprint blueprint) {
+ this.blueprint = blueprint;
+ this.stack = blueprint.getStack();
+ }
+ @Override
+ public void validateTopology() throws InvalidTopologyException {
+ Collection<HostGroup> hostGroups = blueprint.getHostGroups().values();
+ Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies =
+ new HashMap<String, Map<String, Collection<DependencyInfo>>>();
+
+ Collection<String> services = blueprint.getServices();
+ for (HostGroup group : hostGroups) {
+ Map<String, Collection<DependencyInfo>> missingGroupDependencies = validateHostGroup(group);
+ if (! missingGroupDependencies.isEmpty()) {
+ missingDependencies.put(group.getName(), missingGroupDependencies);
+ }
+ }
+
+ Collection<String> cardinalityFailures = new HashSet<String>();
+ for (String service : services) {
+ for (String component : stack.getComponents(service)) {
+ Cardinality cardinality = stack.getCardinality(component);
+ AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component);
+ if (cardinality.isAll()) {
+ cardinalityFailures.addAll(verifyComponentInAllHostGroups(component, autoDeploy));
+ } else {
+ cardinalityFailures.addAll(verifyComponentCardinalityCount(
+ component, cardinality, autoDeploy));
+ }
+ }
+ }
+
+ if (! missingDependencies.isEmpty() || ! cardinalityFailures.isEmpty()) {
+ generateInvalidTopologyException(missingDependencies, cardinalityFailures);
+ }
+ }
+
+ @Override
+ public void validateRequiredProperties() throws InvalidTopologyException {
+ //todo: combine with RequiredPasswordValidator
+ Map<String, Map<String, Collection<String>>> missingProperties =
+ new HashMap<String, Map<String, Collection<String>>>();
+
+ // we don't want to include default stack properties so we can't just use hostGroup full properties
+ Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
+
+ for (HostGroup hostGroup : blueprint.getHostGroups().values()) {
+ Collection<String> processedServices = new HashSet<String>();
+ Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>();
+ Map<String, Map<String, String>> operationalConfiguration = new HashMap<String, Map<String, String>>(clusterConfigurations);
+
+ operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties());
+ for (String component : hostGroup.getComponents()) {
+ //check that MYSQL_SERVER component is not available while hive is using existing db
+ if (component.equals("MYSQL_SERVER")) {
+ Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env");
+ if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null
+ && hiveEnvConfig.get("hive_database").startsWith("Existing")) {
+ throw new IllegalArgumentException("Incorrect configuration: MYSQL_SERVER component is available but hive" +
+ " using existing db!");
+ }
+ }
+
+ //for now, AMBARI is not recognized as a service in Stacks
+ if (! component.equals("AMBARI_SERVER")) {
+ String serviceName = stack.getServiceForComponent(component);
+ if (processedServices.add(serviceName)) {
+ Collection<Stack.ConfigProperty> requiredServiceConfigs =
+ stack.getRequiredConfigurationProperties(serviceName);
+
+ for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) {
+ String configCategory = requiredConfig.getType();
+ String propertyName = requiredConfig.getName();
+ if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) {
+ Collection<String> typeRequirements = allRequiredProperties.get(configCategory);
+ if (typeRequirements == null) {
+ typeRequirements = new HashSet<String>();
+ allRequiredProperties.put(configCategory, typeRequirements);
+ }
+ typeRequirements.add(propertyName);
+ }
+ }
+ }
+ }
+ }
+ for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) {
+ String requiredCategory = requiredTypeProperties.getKey();
+ Collection<String> requiredProperties = requiredTypeProperties.getValue();
+ Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ?
+ operationalConfiguration.get(requiredCategory).keySet() :
+ Collections.<String>emptyList();
+
+ requiredProperties.removeAll(operationalTypeProps);
+ if (! requiredProperties.isEmpty()) {
+ String hostGroupName = hostGroup.getName();
+ Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName);
+ if (hostGroupMissingProps == null) {
+ hostGroupMissingProps = new HashMap<String, Collection<String>>();
+ missingProperties.put(hostGroupName, hostGroupMissingProps);
+ }
+ hostGroupMissingProps.put(requiredCategory, requiredProperties);
+ }
+ }
+ }
+
+ if (! missingProperties.isEmpty()) {
+ throw new InvalidTopologyException("Missing required properties. Specify a value for these " +
+ "properties in the blueprint configuration. " + missingProperties);
+ }
+ }
+
+ /**
+ * Verify that a component is included in all host groups.
+ * For components that are auto-install enabled, will add component to topology if needed.
+ *
+ * @param component component to validate
+ * @param autoDeploy auto-deploy information for component
+ *
+ * @return collection of missing component information
+ */
+ private Collection<String> verifyComponentInAllHostGroups(String component, AutoDeployInfo autoDeploy) {
+
+ Collection<String> cardinalityFailures = new HashSet<String>();
+ int actualCount = blueprint.getHostGroupsForComponent(component).size();
+ Map<String, HostGroup> hostGroups = blueprint.getHostGroups();
+ if (actualCount != hostGroups.size()) {
+ if (autoDeploy != null && autoDeploy.isEnabled()) {
+ for (HostGroup group : hostGroups.values()) {
+ group.addComponent(component);
+ }
+ } else {
+ cardinalityFailures.add(component + "(actual=" + actualCount + ", required=ALL)");
+ }
+ }
+ return cardinalityFailures;
+ }
+
+ private Map<String, Collection<DependencyInfo>> validateHostGroup(HostGroup group) {
+ Map<String, Collection<DependencyInfo>> missingDependencies =
+ new HashMap<String, Collection<DependencyInfo>>();
+
+ Collection<String> blueprintServices = blueprint.getServices();
+ Collection<String> groupComponents = group.getComponents();
+ for (String component : new HashSet<String>(groupComponents)) {
+ Collection<DependencyInfo> dependenciesForComponent = stack.getDependenciesForComponent(component);
+ for (DependencyInfo dependency : dependenciesForComponent) {
+ String conditionalService = stack.getConditionalServiceForDependency(dependency);
+ if (conditionalService != null && ! blueprintServices.contains(conditionalService)) {
+ continue;
+ }
+
+ String dependencyScope = dependency.getScope();
+ String componentName = dependency.getComponentName();
+ AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy();
+ boolean resolved = false;
+
+ if (dependencyScope.equals("cluster")) {
+ Collection<String> missingDependencyInfo = verifyComponentCardinalityCount(
+ componentName, new Cardinality("1+"), autoDeployInfo);
+
+ resolved = missingDependencyInfo.isEmpty();
+ } else if (dependencyScope.equals("host")) {
+ if (groupComponents.contains(component) || (autoDeployInfo != null && autoDeployInfo.isEnabled())) {
+ resolved = true;
+ group.addComponent(componentName);
+ }
+ }
+
+ if (! resolved) {
+ Collection<DependencyInfo> missingCompDependencies = missingDependencies.get(component);
+ if (missingCompDependencies == null) {
+ missingCompDependencies = new HashSet<DependencyInfo>();
+ missingDependencies.put(component, missingCompDependencies);
+ }
+ missingCompDependencies.add(dependency);
+ }
+ }
+ }
+ return missingDependencies;
+ }
+
+ /**
+ * Verify that a component meets cardinality requirements. For components that are
+ * auto-install enabled, will add component to topology if needed.
+ *
+ * @param component component to validate
+ * @param cardinality required cardinality
+ * @param autoDeploy auto-deploy information for component
+ *
+ * @return collection of missing component information
+ */
+ public Collection<String> verifyComponentCardinalityCount(String component,
+ Cardinality cardinality,
+ AutoDeployInfo autoDeploy) {
+
+ Map<String, Map<String, String>> configProperties = blueprint.getConfiguration().getProperties();
+ Collection<String> cardinalityFailures = new HashSet<String>();
+ //todo: don't hard code this HA logic here
+ if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) &&
+ (component.equals("SECONDARY_NAMENODE"))) {
+ // override the cardinality for this component in an HA deployment,
+ // since the SECONDARY_NAMENODE should not be started in this scenario
+ cardinality = new Cardinality("0");
+ }
+
+ int actualCount = blueprint.getHostGroupsForComponent(component).size();
+ if (! cardinality.isValidCount(actualCount)) {
+ boolean validated = ! isDependencyManaged(stack, component, configProperties);
+ if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) {
+ String coLocateName = autoDeploy.getCoLocate();
+ if (coLocateName != null && ! coLocateName.isEmpty()) {
+ Collection<HostGroup> coLocateHostGroups = blueprint.getHostGroupsForComponent(coLocateName.split("/")[1]);
+ if (! coLocateHostGroups.isEmpty()) {
+ validated = true;
+ HostGroup group = coLocateHostGroups.iterator().next();
+ group.addComponent(component);
+
+ }
+ }
+ }
+ if (! validated) {
+ cardinalityFailures.add(component + "(actual=" + actualCount + ", required=" +
+ cardinality.getValue() + ")");
+ }
+ }
+ return cardinalityFailures;
+ }
+
+ /**
+ * Determine if a component is managed, meaning that it is running inside of the cluster
+ * topology. Generally, non-managed dependencies will be database components.
+ *
+ * @param stack stack instance
+ * @param component component to determine if it is managed
+ * @param clusterConfig cluster configuration
+ *
+ * @return true if the specified component managed by the cluster; false otherwise
+ */
+ protected boolean isDependencyManaged(Stack stack, String component, Map<String, Map<String, String>> clusterConfig) {
+ boolean isManaged = true;
+ String externalComponentConfig = stack.getExternalComponentConfig(component);
+ if (externalComponentConfig != null) {
+ String[] toks = externalComponentConfig.split("/");
+ String externalComponentConfigType = toks[0];
+ String externalComponentConfigProp = toks[1];
+ Map<String, String> properties = clusterConfig.get(externalComponentConfigType);
+ if (properties != null && properties.containsKey(externalComponentConfigProp)) {
+ if (properties.get(externalComponentConfigProp).startsWith("Existing")) {
+ isManaged = false;
+ }
+ }
+ }
+ return isManaged;
+ }
+
+ /**
+ * Generate an exception for topology validation failure.
+ *
+ * @param missingDependencies missing dependency information
+ * @param cardinalityFailures missing service component information
+ *
+ * @throws IllegalArgumentException Always thrown and contains information regarding the topology validation failure
+ * in the msg
+ */
+ private void generateInvalidTopologyException(Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies,
+ Collection<String> cardinalityFailures) throws InvalidTopologyException {
+
+ //todo: encapsulate some of this in exception?
+ String msg = "Cluster Topology validation failed.";
+ if (! cardinalityFailures.isEmpty()) {
+ msg += " Invalid service component count: " + cardinalityFailures;
+ }
+ if (! missingDependencies.isEmpty()) {
+ msg += " Unresolved component dependencies: " + missingDependencies;
+ }
+ msg += ". To disable topology validation and create the blueprint, " +
+ "add the following to the end of the url: '?validate_topology=false'";
+ throw new InvalidTopologyException(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java
new file mode 100644
index 0000000..666b1bd
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Cardinality.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+/**
+ * Component cardinality representation.
+ */
+public class Cardinality {
+ String cardinality;
+ int min = 0;
+ int max = Integer.MAX_VALUE;
+ int exact = -1;
+ boolean isAll = false;
+
+ public Cardinality(String cardinality) {
+ this.cardinality = cardinality;
+ if (cardinality != null && ! cardinality.isEmpty()) {
+ if (cardinality.contains("+")) {
+ min = Integer.valueOf(cardinality.split("\\+")[0]);
+ } else if (cardinality.contains("-")) {
+ String[] toks = cardinality.split("-");
+ min = Integer.parseInt(toks[0]);
+ max = Integer.parseInt(toks[1]);
+ } else if (cardinality.equals("ALL")) {
+ isAll = true;
+ } else {
+ exact = Integer.parseInt(cardinality);
+ }
+ }
+ }
+
+ /**
+ * Determine if component is required for all host groups.
+ *
+ * @return true if cardinality is 'ALL', false otherwise
+ */
+ public boolean isAll() {
+ return isAll;
+ }
+
+ /**
+ * Determine if the given count satisfies the required cardinality.
+ *
+ * @param count number of host groups containing component
+ *
+ * @return true id count satisfies the required cardinality, false otherwise
+ */
+ public boolean isValidCount(int count) {
+ if (isAll) {
+ return false;
+ } else if (exact != -1) {
+ return count == exact;
+ } else return count >= min && count <= max;
+ }
+
+ /**
+ * Determine if the cardinality count supports auto-deployment.
+ * This determination is independent of whether the component is configured
+ * to be auto-deployed. This only indicates whether auto-deployment is
+ * supported for the current cardinality.
+ *
+ * At this time, only cardinalities of ALL or where a count of 1 is valid are
+ * supported.
+ *
+ * @return true if cardinality supports auto-deployment
+ */
+ public boolean supportsAutoDeploy() {
+ return isValidCount(1) || isAll;
+ }
+
+ public String getValue() {
+ return cardinality;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
new file mode 100644
index 0000000..1bffbf2
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariServer;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigurationRequest;
+import org.apache.ambari.server.controller.internal.AbstractResourceProvider;
+import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
+import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
+import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.SecurityType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Responsible for cluster configuration.
+ */
+public class ClusterConfigurationRequest {
+
+ protected final static Logger LOG = LoggerFactory.getLogger(ClusterConfigurationRequest.class);
+
+ private ClusterTopology clusterTopology;
+ private BlueprintConfigurationProcessor configurationProcessor;
+ private AmbariManagementController controller = AmbariServer.getController();
+ private Stack stack;
+
+ public ClusterConfigurationRequest(ClusterTopology clusterTopology) throws AmbariException {
+ Blueprint blueprint = clusterTopology.getBlueprint();
+ this.stack = blueprint.getStack();
+ this.clusterTopology = clusterTopology;
+ // set initial configuration (not topology resolved)
+ this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology);
+ setConfigurationsOnCluster(clusterTopology, "INITIAL");
+ }
+
+ // get names of required host groups
+ public Collection<String> getRequiredHostGroups() {
+ return configurationProcessor.getRequiredHostGroups();
+ }
+
+ public void process() throws AmbariException, ConfigurationTopologyException {
+ // this will update the topo cluster config and all host group configs in the cluster topology
+ configurationProcessor.doUpdateForClusterCreate();
+ setConfigurationsOnCluster(clusterTopology, "TOPOLOGY_RESOLVED");
+ }
+
+ /**
+ * Set all configurations on the cluster resource.
+ * @param clusterTopology cluster topology
+ * @param tag config tag
+ *
+ * @throws AmbariException unable to set config on cluster
+ */
+ public void setConfigurationsOnCluster(ClusterTopology clusterTopology, String tag) throws AmbariException {
+ //todo: also handle setting of host group scoped configuration which is updated by config processor
+ List<BlueprintServiceConfigRequest> listofConfigRequests = new LinkedList<BlueprintServiceConfigRequest>();
+
+ Blueprint blueprint = clusterTopology.getBlueprint();
+ Configuration clusterConfiguration = clusterTopology.getConfiguration();
+
+ for (String service : blueprint.getServices()) {
+ //todo: remove intermediate request type
+ // one bp config request per service
+ BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service);
+
+ for (String serviceConfigType : stack.getAllConfigurationTypes(service)) {
+ Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(service);
+ if (!excludedConfigTypes.contains(serviceConfigType)) {
+ // skip handling of cluster-env here
+ if (! serviceConfigType.equals("cluster-env")) {
+ if (clusterConfiguration.getFullProperties().containsKey(serviceConfigType)) {
+ blueprintConfigRequest.addConfigElement(serviceConfigType,
+ clusterConfiguration.getFullProperties().get(serviceConfigType),
+ clusterConfiguration.getFullAttributes().get(serviceConfigType));
+ }
+ }
+ }
+ }
+
+ listofConfigRequests.add(blueprintConfigRequest);
+ }
+
+ // since the stack returns "cluster-env" with each service's config ensure that only one
+ // ClusterRequest occurs for the global cluster-env configuration
+ BlueprintServiceConfigRequest globalConfigRequest = new BlueprintServiceConfigRequest("GLOBAL-CONFIG");
+ Map<String, String> clusterEnvProps = clusterConfiguration.getFullProperties().get("cluster-env");
+ Map<String, Map<String, String>> clusterEnvAttributes = clusterConfiguration.getFullAttributes().get("cluster-env");
+
+ globalConfigRequest.addConfigElement("cluster-env", clusterEnvProps,clusterEnvAttributes);
+ listofConfigRequests.add(globalConfigRequest);
+
+ setConfigurationsOnCluster(listofConfigRequests, tag);
+ }
+
+ /**
+ * Creates a ClusterRequest for each service that
+ * includes any associated config types and configuration. The Blueprints
+ * implementation will now create one ClusterRequest per service, in order
+ * to comply with the ServiceConfigVersioning framework in Ambari.
+ *
+ * This method will also send these requests to the management controller.
+ *
+ * @param listOfBlueprintConfigRequests a list of requests to send to the AmbariManagementController.
+ *
+ * @throws AmbariException upon any error that occurs during updateClusters
+ */
+ private void setConfigurationsOnCluster(List<BlueprintServiceConfigRequest> listOfBlueprintConfigRequests,
+ String tag) throws AmbariException {
+ // iterate over services to deploy
+ for (BlueprintServiceConfigRequest blueprintConfigRequest : listOfBlueprintConfigRequests) {
+ ClusterRequest clusterRequest = null;
+ // iterate over the config types associated with this service
+ List<ConfigurationRequest> requestsPerService = new LinkedList<ConfigurationRequest>();
+ for (BlueprintServiceConfigElement blueprintElement : blueprintConfigRequest.getConfigElements()) {
+ Map<String, Object> clusterProperties = new HashMap<String, Object>();
+ clusterProperties.put(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID, clusterTopology.getClusterName());
+ clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/type", blueprintElement.getTypeName());
+ clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/tag", tag);
+ for (Map.Entry<String, String> entry : blueprintElement.getConfiguration().entrySet()) {
+ clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID +
+ "/properties/" + entry.getKey(), entry.getValue());
+ }
+ if (blueprintElement.getAttributes() != null) {
+ for (Map.Entry<String, Map<String, String>> attribute : blueprintElement.getAttributes().entrySet()) {
+ String attributeName = attribute.getKey();
+ for (Map.Entry<String, String> attributeOccurrence : attribute.getValue().entrySet()) {
+ clusterProperties.put(ClusterResourceProvider.CLUSTER_DESIRED_CONFIGS_PROPERTY_ID + "/properties_attributes/"
+ + attributeName + "/" + attributeOccurrence.getKey(), attributeOccurrence.getValue());
+ }
+ }
+ }
+
+ // only create one cluster request per service, which includes
+ // all the configuration types for that service
+ if (clusterRequest == null) {
+ SecurityType securityType;
+ String requestedSecurityType = (String) clusterProperties.get(
+ ClusterResourceProvider.CLUSTER_SECURITY_TYPE_PROPERTY_ID);
+ if(requestedSecurityType == null)
+ securityType = null;
+ else {
+ try {
+ securityType = SecurityType.valueOf(requestedSecurityType.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(String.format(
+ "Cannot set cluster security type to invalid value: %s", requestedSecurityType));
+ }
+ }
+
+ clusterRequest = new ClusterRequest(
+ (Long) clusterProperties.get(ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID),
+ (String) clusterProperties.get(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID),
+ (String) clusterProperties.get(ClusterResourceProvider.CLUSTER_PROVISIONING_STATE_PROPERTY_ID),
+ securityType,
+ (String) clusterProperties.get(ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID),
+ null);
+ }
+
+ //todo: made getConfigurationRequests static so that I could access from here, where does it belong?
+ List<ConfigurationRequest> listOfRequests =
+ AbstractResourceProvider.getConfigurationRequests("Clusters", clusterProperties);
+ requestsPerService.addAll(listOfRequests);
+ }
+
+ // set total list of config requests, including all config types for this service
+ if (clusterRequest != null) {
+ clusterRequest.setDesiredConfig(requestsPerService);
+ LOG.info("Sending cluster config update request for service = " + blueprintConfigRequest.getServiceName());
+ controller.updateClusters(Collections.singleton(clusterRequest), null);
+ } else {
+ LOG.error("ClusterRequest should not be null for service = " + blueprintConfigRequest.getServiceName());
+ }
+ }
+ }
+
+ /**
+ * Internal class meant to represent the collection of configuration
+ * items and configuration attributes that are associated with a given service.
+ *
+ * This class is used to support proper configuration versioning when
+ * Ambari Blueprints is used to deploy a cluster.
+ */
+ private static class BlueprintServiceConfigRequest {
+
+ private final String serviceName;
+
+ private List<BlueprintServiceConfigElement> configElements =
+ new LinkedList<BlueprintServiceConfigElement>();
+
+ BlueprintServiceConfigRequest(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
+ void addConfigElement(String type, Map<String, String> props, Map<String, Map<String, String>> attributes) {
+ if (props == null) {
+ props = Collections.emptyMap();
+ }
+
+ if (attributes == null) {
+ attributes = Collections.emptyMap();
+ }
+ configElements.add(new BlueprintServiceConfigElement(type, props, attributes));
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ List<BlueprintServiceConfigElement> getConfigElements() {
+ return configElements;
+ }
+ }
+
+ /**
+ * Internal class that represents the configuration
+ * and attributes for a given configuration type.
+ */
+ private static class BlueprintServiceConfigElement {
+ private final String typeName;
+
+ private final Map<String, String> configuration;
+
+ private final Map<String, Map<String, String>> attributes;
+
+ BlueprintServiceConfigElement(String type, Map<String, String> props, Map<String, Map<String, String>> attributes) {
+ this.typeName = type;
+ this.configuration = props;
+ this.attributes = attributes;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public Map<String, String> getConfiguration() {
+ return configuration;
+ }
+
+ public Map<String, Map<String, String>> getAttributes() {
+ return attributes;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
new file mode 100644
index 0000000..e924653
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Represents a full cluster topology including all instance information as well as the associated
+ * blueprint which provides all abstract topology information.
+ */
+public interface ClusterTopology {
+
+ /**
+ * Get the name of the cluster.
+ *
+ * @return cluster name
+ */
+ public String getClusterName();
+
+ /**
+ * Get the blueprint associated with the cluster.
+ *
+ * @return assocaited blueprint
+ */
+ public Blueprint getBlueprint();
+
+ /**
+ * Get the cluster scoped configuration for the cluster.
+ * This configuration has the blueprint cluster scoped
+ * configuration set as it's parent.
+ *
+ * @return cluster scoped configuration
+ */
+ public Configuration getConfiguration();
+
+ /**
+ * Get host group information.
+ *
+ * @return map of host group name to host group information
+ */
+ public Map<String, HostGroupInfo> getHostGroupInfo();
+
+ /**
+ * Get the names of all of host groups which contain the specified component.
+ *
+ * @param component component name
+ *
+ * @return collection of host group names which contain the specified component
+ */
+ public Collection<String> getHostGroupsForComponent(String component);
+
+ /**
+ * Get the name of the host group which is mapped to the specified host.
+ *
+ * @param hostname host name
+ *
+ * @return name of the host group which is mapped to the specified host or null if
+ * no group is mapped to the host
+ */
+ public String getHostGroupForHost(String hostname);
+
+ /**
+ * Get all hosts which are mapped to a host group which contains the specified component.
+ * The host need only to be mapped to the hostgroup, not actually provisioned.
+ *
+ * @param component component name
+ *
+ * @return collection of hosts for the specified component; will not return null
+ */
+ public Collection<String> getHostAssignmentsForComponent(String component);
+
+ /**
+ * Update the existing topology based on the provided topology request.
+ *
+ * @param topologyRequest request modifying the topology
+ *
+ * @throws InvalidTopologyException if the request specified invalid topology information or if
+ * making the requested changes would result in an invalid topology
+ */
+ public void update(TopologyRequest topologyRequest) throws InvalidTopologyException;
+
+ /**
+ * Add a new host to the topology.
+ *
+ * @param hostGroupName name of associated host group
+ * @param host name of host
+ *
+ * @throws InvalidTopologyException if the host being added is already registered to a different host group
+ * @throws NoSuchHostGroupException if the specified host group is invalid
+ */
+ public void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException;
+
+ /**
+ * Determine if NameNode HA is enabled.
+ *
+ * @return true if NameNode HA is enabled; false otherwise
+ */
+ public boolean isNameNodeHAEnabled();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
new file mode 100644
index 0000000..84e90bf
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Represents a cluster topology.
+ * Topology includes the the associated blueprint, cluster configuration and hostgroup -> host mapping.
+ */
+public class ClusterTopologyImpl implements ClusterTopology {
+
+ private String clusterName;
+ //todo: currently topology is only associated with a single bp
+ //todo: this will need to change to allow usage of multiple bp's for the same cluster
+ //todo: for example: provision using bp1 and scale using bp2
+ private Blueprint blueprint;
+ private Configuration configuration;
+ private final Map<String, HostGroupInfo> hostGroupInfoMap =
+ new HashMap<String, HostGroupInfo>();
+
+
+ //todo: will need to convert all usages of hostgroup name to use fully qualified name (BP/HG)
+ //todo: for now, restrict scaling to the same BP
+ public ClusterTopologyImpl(TopologyRequest topologyRequest) throws InvalidTopologyException {
+ this.clusterName = topologyRequest.getClusterName();
+ // provision cluster currently requires that all hostgroups have same BP so it is ok to use root level BP here
+ this.blueprint = topologyRequest.getBlueprint();
+ this.configuration = topologyRequest.getConfiguration();
+
+ registerHostGroupInfo(topologyRequest.getHostGroupInfo());
+
+ validateTopology(topologyRequest.getTopologyValidators());
+ }
+
+ //todo: only used in tests, remove. Validators not invoked when this constructor is used.
+ public ClusterTopologyImpl(String clusterName,
+ Blueprint blueprint,
+ Configuration configuration,
+ Map<String, HostGroupInfo> hostGroupInfo)
+ throws InvalidTopologyException {
+
+ this.clusterName = clusterName;
+ this.blueprint = blueprint;
+ this.configuration = configuration;
+
+ registerHostGroupInfo(hostGroupInfo);
+ }
+
+ @Override
+ public void update(TopologyRequest topologyRequest) throws InvalidTopologyException {
+ registerHostGroupInfo(topologyRequest.getHostGroupInfo());
+ }
+
+ @Override
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ @Override
+ public Blueprint getBlueprint() {
+ return blueprint;
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public Map<String, HostGroupInfo> getHostGroupInfo() {
+ return hostGroupInfoMap;
+ }
+
+ //todo: do we want to return groups with no requested hosts?
+ @Override
+ public Collection<String> getHostGroupsForComponent(String component) {
+ Collection<String> resultGroups = new ArrayList<String>();
+ for (HostGroup group : getBlueprint().getHostGroups().values() ) {
+ if (group.getComponents().contains(component)) {
+ resultGroups.add(group.getName());
+ }
+ }
+ return resultGroups;
+ }
+
+ @Override
+ public String getHostGroupForHost(String hostname) {
+ for (HostGroupInfo groupInfo : hostGroupInfoMap.values() ) {
+ if (groupInfo.getHostNames().contains(hostname)) {
+ // a host can only be associated with a single host group
+ return groupInfo.getHostGroupName();
+ }
+ }
+ return null;
+ }
+
+ //todo: host info?
+ @Override
+ public void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException {
+ if (blueprint.getHostGroup(hostGroupName) == null) {
+ throw new NoSuchHostGroupException("Attempted to add host to non-existing host group: " + hostGroupName);
+ }
+
+ // check for host duplicates
+ String groupContainsHost = getHostGroupForHost(host);
+ // in case of reserved host, hostgroup will already contain host
+ if (groupContainsHost != null && ! hostGroupName.equals(groupContainsHost)) {
+ throw new InvalidTopologyException(String.format(
+ "Attempted to add host '%s' to hostgroup '%s' but it is already associated with hostgroup '%s'.",
+ host, hostGroupName, groupContainsHost));
+ }
+
+ synchronized(hostGroupInfoMap) {
+ HostGroupInfo existingHostGroupInfo = hostGroupInfoMap.get(hostGroupName);
+ if (existingHostGroupInfo == null) {
+ throw new RuntimeException(String.format("An attempt was made to add host '%s' to an unregistered hostgroup '%s'",
+ host, hostGroupName));
+ }
+ // ok to add same host multiple times to same group
+ existingHostGroupInfo.addHost(host);
+ }
+ }
+
+ @Override
+ public Collection<String> getHostAssignmentsForComponent(String component) {
+ //todo: ordering requirements?
+ Collection<String> hosts = new ArrayList<String>();
+ Collection<String> hostGroups = getHostGroupsForComponent(component);
+ for (String group : hostGroups) {
+ hosts.addAll(getHostGroupInfo().get(group).getHostNames());
+ }
+ return hosts;
+ }
+
+ @Override
+ public boolean isNameNodeHAEnabled() {
+ return isNameNodeHAEnabled(configuration.getFullProperties());
+ }
+
+ public static boolean isNameNodeHAEnabled(Map<String, Map<String, String>> configurationProperties) {
+ return configurationProperties.containsKey("hdfs-site") &&
+ configurationProperties.get("hdfs-site").containsKey("dfs.nameservices");
+ }
+
+ private void validateTopology(List<TopologyValidator> validators)
+ throws InvalidTopologyException {
+
+ for (TopologyValidator validator : validators) {
+ validator.validate(this);
+ }
+ }
+
+ private void registerHostGroupInfo(Map<String, HostGroupInfo> groupInfoMap) throws InvalidTopologyException {
+ checkForDuplicateHosts(groupInfoMap);
+ for (HostGroupInfo hostGroupInfo : groupInfoMap.values() ) {
+ String hostGroupName = hostGroupInfo.getHostGroupName();
+ //todo: doesn't support using a different blueprint for update (scaling)
+ HostGroup baseHostGroup = getBlueprint().getHostGroup(hostGroupName);
+ if (baseHostGroup == null) {
+ throw new IllegalArgumentException("Invalid host_group specified: " + hostGroupName +
+ ". All request host groups must have a corresponding host group in the specified blueprint");
+ }
+ //todo: split into two methods
+ HostGroupInfo existingHostGroupInfo = hostGroupInfoMap.get(hostGroupName);
+ if (existingHostGroupInfo == null) {
+ // blueprint host group config
+ Configuration bpHostGroupConfig = baseHostGroup.getConfiguration();
+ // parent config is BP host group config but with parent set to topology cluster scoped config
+ Configuration parentConfiguration = new Configuration(bpHostGroupConfig.getProperties(),
+ bpHostGroupConfig.getAttributes(), getConfiguration());
+
+ hostGroupInfo.getConfiguration().setParentConfiguration(parentConfiguration);
+ hostGroupInfoMap.put(hostGroupName, hostGroupInfo);
+ } else {
+ // Update. Either add hosts or increment request count
+ if (! hostGroupInfo.getHostNames().isEmpty()) {
+ try {
+ // this validates that hosts aren't already registered with groups
+ addHostsToTopology(hostGroupInfo);
+ } catch (NoSuchHostGroupException e) {
+ //todo
+ throw new InvalidTopologyException("Attempted to add hosts to unknown host group: " + hostGroupName);
+ }
+ } else {
+ existingHostGroupInfo.setRequestedCount(
+ existingHostGroupInfo.getRequestedHostCount() + hostGroupInfo.getRequestedHostCount());
+ }
+ //todo: throw exception in case where request attempts to modify HG configuration in scaling operation
+ }
+ }
+ }
+
+ private void addHostsToTopology(HostGroupInfo hostGroupInfo) throws InvalidTopologyException, NoSuchHostGroupException {
+ for (String host: hostGroupInfo.getHostNames()) {
+ addHostToTopology(hostGroupInfo.getHostGroupName(), host);
+ }
+ }
+
+ private void checkForDuplicateHosts(Map<String, HostGroupInfo> groupInfoMap) throws InvalidTopologyException {
+ Set<String> hosts = new HashSet<String>();
+ Set<String> duplicates = new HashSet<String>();
+ for (HostGroupInfo group : groupInfoMap.values()) {
+ // check for duplicates within the new groups
+ Collection<String> groupHosts = group.getHostNames();
+ Collection<String> groupHostsCopy = new HashSet<String>(group.getHostNames());
+ groupHostsCopy.retainAll(hosts);
+ duplicates.addAll(groupHostsCopy);
+ hosts.addAll(groupHosts);
+
+ // check against existing groups
+ for (String host : groupHosts) {
+ if (getHostGroupForHost(host) != null) {
+ duplicates.add(host);
+ }
+ }
+ }
+ if (! duplicates.isEmpty()) {
+ throw new InvalidTopologyException("The following hosts are mapped to multiple host groups: " + duplicates);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
new file mode 100644
index 0000000..2447b8b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+/**
+ * Configuration for a topology entity such as a blueprint, hostgroup or cluster.
+ */
+public class Configuration {
+
+ private Map<String, Map<String, String>> properties;
+ private Map<String, Map<String, Map<String, String>>> attributes;
+
+ private Configuration parentConfiguration;
+
+ public Configuration(Map<String, Map<String, String>> properties,
+ Map<String, Map<String, Map<String, String>>> attributes,
+ Configuration parentConfiguration) {
+
+ this.properties = properties;
+ this.attributes = attributes;
+ this.parentConfiguration = parentConfiguration;
+
+ //todo: warning for deprecated global properties
+ // String message = null;
+// for (BlueprintConfigEntity blueprintConfig: blueprint.getConfigurations()){
+// if(blueprintConfig.getType().equals("global")){
+// message = "WARNING: Global configurations are deprecated, please use *-env";
+// break;
+// }
+// }
+ }
+
+ public Configuration(Map<String, Map<String, String>> properties,
+ Map<String, Map<String, Map<String, String>>> attributes) {
+
+ this.properties = properties;
+ this.attributes = attributes;
+ }
+
+ public Map<String, Map<String, String>> getProperties() {
+ return properties;
+ }
+
+ public Map<String, Map<String, String>> getFullProperties() {
+ return getFullProperties(Integer.MAX_VALUE);
+ }
+
+ //re-calculated each time in case parent properties changed
+ public Map<String, Map<String, String>> getFullProperties(int depthLimit) {
+
+ if (depthLimit == 0) {
+ return new HashMap<String, Map<String, String>>(properties);
+ }
+
+ Map<String, Map<String, String>> mergedProperties = parentConfiguration == null ?
+ new HashMap<String, Map<String, String>>() :
+ new HashMap<String, Map<String, String>>(parentConfiguration.getFullProperties(--depthLimit));
+
+ for (Map.Entry<String, Map<String, String>> entry : properties.entrySet()) {
+ String configType = entry.getKey();
+ Map<String, String> typeProps = entry.getValue();
+
+ if (mergedProperties.containsKey(configType)) {
+ mergedProperties.get(configType).putAll(typeProps);
+ } else {
+ mergedProperties.put(configType, typeProps);
+ }
+ }
+ return mergedProperties;
+ }
+
+ public Map<String, Map<String, Map<String, String>>> getAttributes() {
+ return attributes;
+ }
+
+ //re-calculate each time in case parent properties changed
+ // attribute structure is very confusing. {type -> {attributeName -> {propName, attributeValue}}}
+ public Map<String, Map<String, Map<String, String>>> getFullAttributes() {
+ Map<String, Map<String, Map<String, String>>> mergedAttributeMap = parentConfiguration == null ?
+ new HashMap<String, Map<String, Map<String, String>>>() :
+ new HashMap<String, Map<String, Map<String, String>>>(parentConfiguration.getFullAttributes());
+
+ for (Map.Entry<String, Map<String, Map<String, String>>> typeEntry : attributes.entrySet()) {
+ String type = typeEntry.getKey();
+ if (! mergedAttributeMap.containsKey(type)) {
+ mergedAttributeMap.put(type, typeEntry.getValue());
+ } else {
+ Map<String, Map<String, String>> mergedAttributes = mergedAttributeMap.get(type);
+ for (Map.Entry<String, Map<String, String>> attributeEntry : typeEntry.getValue().entrySet()) {
+ String attribute = attributeEntry.getKey();
+ if (! mergedAttributes.containsKey(attribute)) {
+ mergedAttributes.put(attribute, attributeEntry.getValue());
+ } else {
+ Map<String, String> mergedAttributeProps = mergedAttributes.get(attribute);
+ for (Map.Entry<String, String> propEntry : attributeEntry.getValue().entrySet()) {
+ mergedAttributeProps.put(propEntry.getKey(), propEntry.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ mergedAttributeMap.putAll(attributes);
+
+ return mergedAttributeMap;
+ }
+
+ public Collection<String> getAllConfigTypes() {
+ Collection<String> allTypes = new HashSet<String>();
+ for (String type : getFullProperties().keySet()) {
+ allTypes.add(type);
+ }
+
+ for (String type : getFullAttributes().keySet()) {
+ allTypes.add(type);
+ }
+
+ return allTypes;
+ }
+
+ public Configuration getParentConfiguration() {
+ return parentConfiguration;
+ }
+
+ public void setParentConfiguration(Configuration parent) {
+ parentConfiguration = parent;
+ }
+
+ public String getPropertyValue(String configType, String propertyName) {
+ return properties.containsKey(configType) ?
+ properties.get(configType).get(propertyName) : null;
+ }
+
+ public boolean containsProperty(String configType, String propertyName) {
+ return properties.containsKey(configType) && properties.get(configType).containsKey(propertyName);
+ }
+
+ public String setProperty(String configType, String propertyName, String value) {
+ Map<String, String> typeProperties = properties.get(configType);
+ if (typeProperties == null) {
+ typeProperties = new HashMap<String, String>();
+ properties.put(configType, typeProperties);
+ }
+
+ return typeProperties.put(propertyName, value);
+ }
+
+ // attribute structure is very confusing: {type -> {attributeName -> {propName, attributeValue}}}
+ public String setAttribute(String configType, String propertyName, String attributeName, String attributeValue) {
+ Map<String, Map<String, String>> typeAttributes = attributes.get(configType);
+ if (typeAttributes == null) {
+ typeAttributes = new HashMap<String, Map<String, String>>();
+ attributes.put(configType, typeAttributes);
+ }
+
+ Map<String, String> attributes = typeAttributes.get(attributeName);
+ if (attributes == null) {
+ attributes = new HashMap<String, String>();
+ typeAttributes.put(attributeName, attributes);
+ }
+
+ return attributes.put(propertyName, attributeValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java
new file mode 100644
index 0000000..f4dc879
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ConfigurationFactory.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Creates a configuration instance from user provided properties.
+ * Supports both forms of configuration syntax.
+ * todo: document both forms here
+*/
+public class ConfigurationFactory {
+
+ private static final String SCHEMA_IS_NOT_SUPPORTED_MESSAGE =
+ "Provided configuration format is not supported";
+
+ public Configuration getConfiguration(Collection<Map<String, String>> configProperties) {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>();
+ Configuration configuration = new Configuration(properties, attributes);
+
+ if (configProperties != null) {
+ for (Map<String, String> typeMap : configProperties) {
+ //todo: can we have a different strategy for each type?
+ ConfigurationStrategy strategy = decidePopulationStrategy(typeMap);
+ for (Map.Entry<String, String> entry : typeMap.entrySet()) {
+ String[] propertyNameTokens = entry.getKey().split("/");
+ strategy.setConfiguration(configuration, propertyNameTokens, entry.getValue());
+ }
+ }
+ }
+ return configuration;
+ }
+
+ private ConfigurationStrategy decidePopulationStrategy(Map<String, String> configuration) {
+ if (configuration != null && !configuration.isEmpty()) {
+ String keyEntry = configuration.keySet().iterator().next();
+ String[] keyNameTokens = keyEntry.split("/");
+ int levels = keyNameTokens.length;
+ String propertiesType = keyNameTokens[1];
+ if (levels == 2) {
+ return new ConfigurationStrategyV1();
+ } else if ((levels == 3 && BlueprintFactory.PROPERTIES_PROPERTY_ID.equals(propertiesType))
+ || (levels == 4 && BlueprintFactory.PROPERTIES_ATTRIBUTES_PROPERTY_ID.equals(propertiesType))) {
+ return new ConfigurationStrategyV2();
+ } else {
+ throw new IllegalArgumentException(SCHEMA_IS_NOT_SUPPORTED_MESSAGE);
+ }
+ } else {
+ return new ConfigurationStrategyV2();
+ }
+ }
+
+ /**
+ * The structure of blueprints is evolving where multiple resource
+ * structures are to be supported. This class abstracts the population
+ * of configurations which have changed from a map of key-value strings,
+ * to an map containing 'properties' and 'properties_attributes' maps.
+ *
+ * Extending classes can determine how they want to populate the
+ * configuration maps depending on input.
+ */
+ private static abstract class ConfigurationStrategy {
+
+ protected abstract void setConfiguration(Configuration configuration,
+ String[] propertyNameTokens,
+ String propertyValue);
+
+ }
+
+ /**
+ * Original blueprint configuration format where configs were a map
+ * of strings.
+ */
+ protected static class ConfigurationStrategyV1 extends ConfigurationStrategy {
+
+
+ @Override
+ protected void setConfiguration(Configuration configuration, String[] propertyNameTokens, String propertyValue) {
+ configuration.setProperty(propertyNameTokens[0], propertyNameTokens[1], propertyValue);
+ }
+ }
+
+ /**
+ * New blueprint configuration format where configs are a map from 'properties' and
+ * 'properties_attributes' to a map of strings.
+ *
+ * @since 1.7.0
+ */
+ protected static class ConfigurationStrategyV2 extends ConfigurationStrategy {
+
+ @Override
+ protected void setConfiguration(Configuration configuration, String[] propertyNameTokens, String propertyValue) {
+ String type = propertyNameTokens[0];
+ if (BlueprintFactory.PROPERTIES_PROPERTY_ID.equals(propertyNameTokens[1])) {
+ configuration.setProperty(type, propertyNameTokens[2], propertyValue);
+ } else if (BlueprintFactory.PROPERTIES_ATTRIBUTES_PROPERTY_ID.equals(propertyNameTokens[1])) {
+ configuration.setAttribute(type, propertyNameTokens[2], propertyNameTokens[3], propertyValue);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java
new file mode 100644
index 0000000..07e3e88
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.orm.entities.HostGroupEntity;
+import org.apache.ambari.server.state.DependencyInfo;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Host Group representation.
+ */
+public interface HostGroup {
+
+ /**
+ * Get the name of the host group.
+ *
+ * @return the host group name
+ */
+ public String getName();
+
+ /**
+ * Get the name of the associated blueprint
+ *
+ * @return associated blueprint name
+ */
+ public String getBlueprintName();
+
+ /**
+ * Get the fully qualified host group name in the form of
+ * blueprintName:hostgroupName
+ *
+ * @return fully qualified host group name
+ */
+ public String getFullyQualifiedName();
+
+ /**
+ * Get all of the host group components.
+ *
+ * @return collection of component names
+ */
+ public Collection<String> getComponents();
+
+ /**
+ * Get the host group components which belong to the specified service.
+ *
+ * @param service service name
+ *
+ * @return collection of component names for the specified service; will not return null
+ */
+ public Collection<String> getComponents(String service);
+
+ /**
+ * Add a component to the host group.
+ *
+ * @param component name of the component to add
+ *
+ * @return true if the component didn't already exist
+ */
+ public boolean addComponent(String component);
+
+ /**
+ * Determine if the host group contains a master component.
+ *
+ * @return true if the host group contains a master component; false otherwise
+ */
+ public boolean containsMasterComponent();
+
+ /**
+ * Get all of the services associated with the host group components.
+ *
+ * @return collection of service names
+ */
+ public Collection<String> getServices();
+
+ /**
+ * Get the configuration associated with the host group.
+ * The host group configuration has the blueprint cluster scoped
+ * configuration set as it's parent.
+ *
+ * @return host group configuration
+ */
+ public Configuration getConfiguration();
+
+ /**
+ * Get the stack associated with the host group.
+ *
+ * @return associated stack
+ */
+ public Stack getStack();
+
+ /**
+ * Get the cardinality value that was specified for the host group.
+ * This is simply meta-data for the stack that a deployer can use
+ * and this information is not used by ambari.
+ *
+ * @return the cardinality specified for the hostgroup
+ */
+ public String getCardinality();
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java
new file mode 100644
index 0000000..b89e7e4
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import com.google.gson.Gson;
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
+import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
+import org.apache.ambari.server.orm.entities.HostGroupEntity;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Host Group implementation.
+ */
+public class HostGroupImpl implements HostGroup {
+
+ /**
+ * host group name
+ */
+ private String name;
+
+ /**
+ * blueprint name
+ */
+ private String blueprintName;
+
+ /**
+ * components contained in the host group
+ */
+ private Collection<String> components = new HashSet<String>();
+
+ /**
+ * map of service to components for the host group
+ */
+ private Map<String, Set<String>> componentsForService = new HashMap<String, Set<String>>();
+
+ /**
+ * configuration
+ */
+ private Configuration configuration = null;
+
+ private boolean containsMasterComponent = false;
+
+ private Stack stack;
+
+ private String cardinality = "NOT SPECIFIED";
+
+ public HostGroupImpl(HostGroupEntity entity, String blueprintName, Stack stack) {
+ this.name = entity.getName();
+ this.cardinality = entity.getCardinality();
+ this.blueprintName = blueprintName;
+ this.stack = stack;
+
+ parseComponents(entity);
+ parseConfigurations(entity);
+ }
+
+ public HostGroupImpl(String name, String bpName, Stack stack, Collection<String> components, Configuration configuration, String cardinality) {
+ this.name = name;
+ this.blueprintName = bpName;
+ this.stack = stack;
+
+ // process each component
+ for (String component : components) {
+ addComponent(component);
+ }
+ this.configuration = configuration;
+ if (cardinality != null && ! cardinality.equals("null")) {
+ this.cardinality = cardinality;
+ }
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ //todo: currently not qualifying host group name
+ @Override
+ public String getFullyQualifiedName() {
+ return String.format("%s:%s", blueprintName, getName());
+ }
+
+ //todo: currently not qualifying host group name
+ public static String formatAbsoluteName(String bpName, String hgName) {
+ return String.format("%s:%s", bpName, hgName);
+ }
+
+ @Override
+ public Collection<String> getComponents() {
+ return components;
+ }
+
+ /**
+ * Get the services which are deployed to this host group.
+ *
+ * @return collection of services which have components in this host group
+ */
+ @Override
+ public Collection<String> getServices() {
+ return componentsForService.keySet();
+ }
+
+ /**
+ * Add a component to the host group.
+ *
+ * @param component component to add
+ *
+ * @return true if component was added; false if component already existed
+ */
+ @Override
+ public boolean addComponent(String component) {
+ boolean added = components.add(component);
+ if (stack.isMasterComponent(component)) {
+ containsMasterComponent = true;
+ }
+ if (added) {
+ String service = stack.getServiceForComponent(component);
+ if (service != null) {
+ // an example of a component without a service in the stack is AMBARI_SERVER
+ Set<String> serviceComponents = componentsForService.get(service);
+ if (serviceComponents == null) {
+ serviceComponents = new HashSet<String>();
+ componentsForService.put(service, serviceComponents);
+ }
+ serviceComponents.add(component);
+ }
+ }
+ return added;
+ }
+
+ /**
+ * Get the components for the specified service which are associated with the host group.
+ *
+ * @param service service name
+ *
+ * @return set of component names
+ */
+ @Override
+ public Collection<String> getComponents(String service) {
+ return componentsForService.containsKey(service) ?
+ new HashSet<String>(componentsForService.get(service)) :
+ Collections.<String>emptySet();
+ }
+
+ /**
+ * Get this host groups configuration.
+ *
+ * @return configuration instance
+ */
+ @Override
+ public Configuration getConfiguration() {
+
+ return configuration;
+ }
+
+ /**
+ * Get the associated blueprint name.
+ *
+ * @return associated blueprint name
+ */
+ @Override
+ public String getBlueprintName() {
+ return blueprintName;
+ }
+
+ @Override
+ public boolean containsMasterComponent() {
+ return containsMasterComponent;
+ }
+
+ @Override
+ public Stack getStack() {
+ return stack;
+ }
+
+ @Override
+ public String getCardinality() {
+ return cardinality;
+ }
+
+ /**
+ * Parse component information.
+ */
+ private void parseComponents(HostGroupEntity entity) {
+ for (HostGroupComponentEntity componentEntity : entity.getComponents() ) {
+ addComponent(componentEntity.getName());
+ }
+ }
+
+ /**
+ * Parse host group configurations.
+ */
+ //todo: use ConfigurationFactory
+ private void parseConfigurations(HostGroupEntity entity) {
+ Map<String, Map<String, String>> config = new HashMap<String, Map<String, String>>();
+ Gson jsonSerializer = new Gson();
+ for (HostGroupConfigEntity configEntity : entity.getConfigurations()) {
+ String type = configEntity.getType();
+ Map<String, String> typeProperties = config.get(type);
+ if ( typeProperties == null) {
+ typeProperties = new HashMap<String, String>();
+ config.put(type, typeProperties);
+ }
+ Map<String, String> propertyMap = jsonSerializer.<Map<String, String>>fromJson(
+ configEntity.getConfigData(), Map.class);
+
+ if (propertyMap != null) {
+ typeProperties.putAll(propertyMap);
+ }
+ }
+ //todo: parse attributes
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>();
+ configuration = new Configuration(config, attributes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
new file mode 100644
index 0000000..07cc1b2
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.spi.Predicate;
+
+import java.util.Collection;
+import java.util.HashSet;
+
+/**
+ * Host Group information specific to a cluster instance.
+ */
+public class HostGroupInfo {
+
+ private String hostGroupName;
+ /**
+ * Hosts contained associated with the host group
+ */
+ private Collection<String> hostNames = new HashSet<String>();
+
+ private int requested_count = 0;
+
+ Configuration configuration;
+
+
+ Predicate predicate;
+
+
+ public HostGroupInfo(String hostGroupName) {
+ this.hostGroupName = hostGroupName;
+ }
+
+ public String getHostGroupName() {
+ return hostGroupName;
+ }
+
+ public Collection<String> getHostNames() {
+ return new HashSet<String>(hostNames);
+ }
+
+ public int getRequestedHostCount() {
+ return requested_count == 0 ? hostNames.size() : requested_count;
+ }
+
+ public void addHost(String hostName) {
+ hostNames.add(hostName);
+ }
+
+ public void addHosts(Collection<String> hosts) {
+ for (String host : hosts) {
+ addHost(host);
+ }
+ }
+
+ public void setRequestedCount(int num) {
+ requested_count = num;
+ }
+
+ //todo: constructor?
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setPredicate(Predicate predicate) {
+ this.predicate = predicate;
+ }
+
+ public Predicate getPredicate() {
+ return predicate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java
new file mode 100644
index 0000000..ce636e2
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostOfferResponse.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.List;
+
+/**
+ * Response to a host offer.
+ */
+public class HostOfferResponse {
+ public enum Answer {ACCEPTED, DECLINED_PREDICATE, DECLINED_DONE}
+
+ private final Answer answer;
+ private final String hostGroupName;
+ private final List<TopologyTask> tasks;
+
+ public HostOfferResponse(Answer answer) {
+ if (answer == Answer.ACCEPTED) {
+ throw new IllegalArgumentException("For accepted response, hostgroup name and tasks must be set");
+ }
+ this.answer = answer;
+ this.hostGroupName = null;
+ this.tasks = null;
+ }
+
+ public HostOfferResponse(Answer answer, String hostGroupName, List<TopologyTask> tasks) {
+ this.answer = answer;
+ this.hostGroupName = hostGroupName;
+ this.tasks = tasks;
+ }
+
+ public Answer getAnswer() {
+ return answer;
+ }
+
+ //todo: for now assumes a host was added
+ //todo: perhaps a topology modification object that modifies a passed in topology structure?
+ public String getHostGroupName() {
+ return hostGroupName;
+ }
+
+ public List<TopologyTask> getTasks() {
+ return tasks;
+ }
+}