You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by js...@apache.org on 2015/04/27 07:53:06 UTC
[12/13] ambari git commit: AMBARI-10750. Initial merge of advanced
api provisioning work.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
deleted file mode 100644
index 73ea1a5..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
+++ /dev/null
@@ -1,771 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.controller.internal;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.StackAccessException;
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
-import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
-import org.apache.ambari.server.controller.spi.ResourceProvider;
-import org.apache.ambari.server.controller.spi.SystemException;
-import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
-import org.apache.ambari.server.orm.dao.BlueprintDAO;
-import org.apache.ambari.server.orm.dao.StackDAO;
-import org.apache.ambari.server.orm.entities.BlueprintConfigEntity;
-import org.apache.ambari.server.orm.entities.BlueprintEntity;
-import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
-import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
-import org.apache.ambari.server.orm.entities.HostGroupEntity;
-import org.apache.ambari.server.state.AutoDeployInfo;
-import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.state.DependencyInfo;
-
-import com.google.gson.Gson;
-
-/**
- * Base blueprint processing resource provider.
- */
-//todo: this class needs to be refactored to a ClusterTopology class which
-//todo: has hostgroup, stack and configuration state specific to a deployment.
-public abstract class BaseBlueprintProcessor extends AbstractControllerResourceProvider {
-
- /**
- * Data access object used to obtain blueprint entities.
- */
- protected static BlueprintDAO blueprintDAO;
-
- /**
- * Data access object used to lookup value stacks parsed from the resources.
- */
- protected static StackDAO stackDAO;
-
- /**
- * Stack related information.
- */
- protected static AmbariMetaInfo stackInfo;
-
- protected static ConfigHelper configHelper;
-
-
- protected BaseBlueprintProcessor(Set<String> propertyIds,
- Map<Resource.Type, String> keyPropertyIds,
- AmbariManagementController managementController) {
-
- super(propertyIds, keyPropertyIds, managementController);
- }
-
- /**
- * Get host groups which contain a component.
- *
- * @param component component name
- * @param hostGroups collection of host groups to check
- *
- * @return collection of host groups which contain the specified component
- */
- protected Collection<HostGroupImpl> getHostGroupsForComponent(String component, Collection<HostGroupImpl> hostGroups) {
- Collection<HostGroupImpl> resultGroups = new HashSet<HostGroupImpl>();
- for (HostGroupImpl group : hostGroups ) {
- if (group.getComponents().contains(component)) {
- resultGroups.add(group);
- }
- }
- return resultGroups;
- }
-
- /**
- * Parse blueprint host groups.
- *
- * @param blueprint associated blueprint
- * @param stack associated stack
- *
- * @return map of host group name to host group
- */
- protected Map<String, HostGroupImpl> parseBlueprintHostGroups(BlueprintEntity blueprint, Stack stack) {
- Map<String, HostGroupImpl> mapHostGroups = new HashMap<String, HostGroupImpl>();
-
- for (HostGroupEntity hostGroup : blueprint.getHostGroups()) {
- mapHostGroups.put(hostGroup.getName(), new HostGroupImpl(hostGroup, stack, this));
- }
- return mapHostGroups;
- }
-
- /**
- * Parse stack information.
- *
- * @param blueprint associated blueprint
- *
- * @return stack instance
- *
- * @throws SystemException an unexpected exception occurred
- */
- protected Stack parseStack(BlueprintEntity blueprint) throws SystemException {
- Stack stack;
- try {
- stack = new Stack(blueprint.getStack(), getManagementController());
- } catch (StackAccessException e) {
- throw new IllegalArgumentException(
- "Invalid stack information provided for cluster. "
- + blueprint.getStack());
- } catch (AmbariException e) {
- throw new SystemException("Unable to obtain stack information.", e);
- }
- return stack;
- }
-
- /**
- * Validate blueprint topology.
- * An exception is thrown in the case of validation failure.
- * For missing components which are auto-deploy enabled, these are added to the topology which is reflected
- * in the blueprint entity that is returned.
- *
- * @param blueprint blueprint to validate
- *
- * @return blueprint entity which may have been updated as a result of auto-deployment of components.
- *
- * @throws AmbariException an unexpected error occurred
- * @throws IllegalArgumentException when validation fails
- */
- protected BlueprintEntity validateTopology(BlueprintEntity blueprint) throws AmbariException {
- Stack stack = new Stack(blueprint.getStack(), getManagementController());
- Map<String, HostGroupImpl> hostGroupMap = parseBlueprintHostGroups(blueprint, stack);
- Collection<HostGroupImpl> hostGroups = hostGroupMap.values();
- Map<String, Map<String, String>> clusterConfig = processBlueprintConfigurations(blueprint, null);
- Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies =
- new HashMap<String, Map<String, Collection<DependencyInfo>>>();
-
- Collection<String> services = getTopologyServices(hostGroups);
- for (HostGroupImpl group : hostGroups) {
- Map<String, Collection<DependencyInfo>> missingGroupDependencies =
- group.validateTopology(hostGroups, services, clusterConfig);
- if (! missingGroupDependencies.isEmpty()) {
- missingDependencies.put(group.getEntity().getName(), missingGroupDependencies);
- }
- }
-
- Collection<String> cardinalityFailures = new HashSet<String>();
- for (String service : services) {
- for (String component : stack.getComponents(service)) {
- Cardinality cardinality = stack.getCardinality(component);
- AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component);
- if (cardinality.isAll()) {
- cardinalityFailures.addAll(verifyComponentInAllHostGroups(
- blueprint, hostGroups, component, autoDeploy));
- } else {
- cardinalityFailures.addAll(verifyComponentCardinalityCount(
- blueprint, hostGroups, component, cardinality, autoDeploy, stack, clusterConfig));
- }
- }
- }
-
- if (! missingDependencies.isEmpty() || ! cardinalityFailures.isEmpty()) {
- generateInvalidTopologyException(missingDependencies, cardinalityFailures);
- }
-
- return blueprint;
- }
-
- /**
- * Process cluster scoped configurations contained in blueprint.
- *
- * @param blueprint blueprint entity
- *
- * @return cluster scoped properties contained within in blueprint
- */
- protected Map<String, Map<String, String>> processBlueprintConfigurations(
- BlueprintEntity blueprint, Collection<Map<String, String>> configOverrides) {
-
- Map<String, Map<String, String>> mapConfigurations = new HashMap<String, Map<String, String>>();
- Collection<BlueprintConfigEntity> configs = blueprint.getConfigurations();
- Gson jsonSerializer = new Gson();
-
- for (BlueprintConfigEntity config : configs) {
- mapConfigurations.put(config.getType(), jsonSerializer.<Map<String, String>> fromJson(
- config.getConfigData(), Map.class));
- }
- overrideExistingProperties(mapConfigurations, configOverrides);
-
- return mapConfigurations;
- }
-
- /**
- * Process cluster scoped configuration attributes contained in blueprint.
- *
- * @param blueprint blueprint entity
- *
- * @return cluster scoped property attributes contained within in blueprint
- */
- protected Map<String, Map<String, Map<String, String>>> processBlueprintAttributes(BlueprintEntity blueprint) {
-
- Map<String, Map<String, Map<String, String>>> mapAttributes =
- new HashMap<String, Map<String, Map<String, String>>>();
- Collection<BlueprintConfigEntity> configs = blueprint.getConfigurations();
-
- if (configs != null) {
- Gson gson = new Gson();
- for (BlueprintConfigEntity config : configs) {
- Map<String, Map<String, String>> typeAttrs =
- gson.<Map<String, Map<String, String>>>fromJson(config.getConfigAttributes(), Map.class);
- if (typeAttrs != null && !typeAttrs.isEmpty()) {
- mapAttributes.put(config.getType(), typeAttrs);
- }
- }
- }
-
- return mapAttributes;
- }
-
- /**
- * Override existing properties or add new.
- *
- * @param existingProperties current property values
- * @param configOverrides override properties
- */
- protected void overrideExistingProperties(Map<String, Map<String, String>> existingProperties,
- Collection<Map<String, String>> configOverrides) {
- if (configOverrides != null) {
- for (Map<String, String> properties : configOverrides) {
- String category = null;
- int propertyOffset = -1;
- for (Map.Entry<String, String> entry : properties.entrySet()) {
- String absolutePropName = entry.getKey();
- if (category == null) {
- propertyOffset = absolutePropName.indexOf('/');
- category = absolutePropName.substring(0, propertyOffset);
- }
- Map<String, String> existingCategoryProperties = existingProperties.get(category);
- if (existingCategoryProperties == null) {
- existingCategoryProperties = new HashMap<String, String>();
- existingProperties.put(category, existingCategoryProperties);
- }
- //override existing property or add new
- existingCategoryProperties.put(absolutePropName.substring(propertyOffset + 1), entry.getValue());
- }
- }
- }
- }
-
- /**
- * Add a new component entity to a host group entity.
- *
- * @param blueprint blueprint entity
- * @param hostGroup host group name
- * @param component name of component which is being added
- */
- protected void addComponentToBlueprint(BlueprintEntity blueprint, String hostGroup, String component) {
- HostGroupComponentEntity componentEntity = new HostGroupComponentEntity();
- componentEntity.setBlueprintName(blueprint.getBlueprintName());
- componentEntity.setName(component);
-
- for (HostGroupEntity hostGroupEntity : blueprint.getHostGroups()) {
- if (hostGroupEntity.getName().equals(hostGroup)) {
- componentEntity.setHostGroupEntity(hostGroupEntity);
- componentEntity.setHostGroupName(hostGroupEntity.getName());
- hostGroupEntity.addComponent(componentEntity);
- break;
- }
- }
- }
-
- /**
- * Obtain a blueprint entity based on name.
- *
- * @param blueprintName name of blueprint to obtain
- *
- * @return blueprint entity for the given name
- * @throws IllegalArgumentException no blueprint with the given name found
- */
- protected BlueprintEntity getExistingBlueprint(String blueprintName) {
- BlueprintEntity blueprint = blueprintDAO.findByName(blueprintName);
- if (blueprint == null) {
- throw new IllegalArgumentException("Specified blueprint doesn't exist: " + blueprintName);
- }
- return blueprint;
- }
-
- /**
- * Get all services provided in topology.
- *
- * @param hostGroups all host groups in topology
- *
- * @return collections of all services provided by topology
- */
- protected Collection<String> getTopologyServices(Collection<HostGroupImpl> hostGroups) {
- Collection<String> services = new HashSet<String>();
- for (HostGroupImpl group : hostGroups) {
- services.addAll(group.getServices());
- }
- return services;
- }
-
- /**
- * Determine if a component is managed, meaning that it is running inside of the cluster
- * topology. Generally, non-managed dependencies will be database components.
- *
- * @param stack stack instance
- * @param component component to determine if it is managed
- * @param clusterConfig cluster configuration
- *
- * @return true if the specified component managed by the cluster; false otherwise
- */
- protected boolean isDependencyManaged(Stack stack, String component, Map<String, Map<String, String>> clusterConfig) {
- boolean isManaged = true;
- String externalComponentConfig = stack.getExternalComponentConfig(component);
- if (externalComponentConfig != null) {
- String[] toks = externalComponentConfig.split("/");
- String externalComponentConfigType = toks[0];
- String externalComponentConfigProp = toks[1];
- Map<String, String> properties = clusterConfig.get(externalComponentConfigType);
- if (properties != null && properties.containsKey(externalComponentConfigProp)) {
- if (properties.get(externalComponentConfigProp).startsWith("Existing")) {
- isManaged = false;
- }
- }
- }
- return isManaged;
- }
-
- /**
- * Verify that a component meets cardinality requirements. For components that are
- * auto-install enabled, will add component to topology if needed.
- *
- * @param blueprint blueprint instance
- * @param hostGroups collection of host groups
- * @param component component to validate
- * @param cardinality required cardinality
- * @param autoDeploy auto-deploy information for component
- *
- * @return collection of missing component information
- */
- private Collection<String> verifyComponentCardinalityCount(BlueprintEntity blueprint,
- Collection<HostGroupImpl> hostGroups,
- String component,
- Cardinality cardinality,
- AutoDeployInfo autoDeploy,
- Stack stack,
- Map<String, Map<String, String>> clusterConfig) {
-
- Collection<String> cardinalityFailures = new HashSet<String>();
-
- if (BlueprintConfigurationProcessor.isNameNodeHAEnabled(clusterConfig) &&
- (component.equals("SECONDARY_NAMENODE"))) {
- // override the cardinality for this component in an HA deployment,
- // since the SECONDARY_NAMENODE should not be started in this scenario
- cardinality = new Cardinality("0");
- }
-
- int actualCount = getHostGroupsForComponent(component, hostGroups).size();
- if (! cardinality.isValidCount(actualCount)) {
- boolean validated = ! isDependencyManaged(stack, component, clusterConfig);
- if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) {
- String coLocateName = autoDeploy.getCoLocate();
- if (coLocateName != null && ! coLocateName.isEmpty()) {
- Collection<HostGroupImpl> coLocateHostGroups = getHostGroupsForComponent(
- coLocateName.split("/")[1], hostGroups);
- if (! coLocateHostGroups.isEmpty()) {
- validated = true;
- HostGroupImpl group = coLocateHostGroups.iterator().next();
- if (group.addComponent(component)) {
- addComponentToBlueprint(blueprint, group.getEntity().getName(), component);
- }
- }
- }
- }
- if (! validated) {
- cardinalityFailures.add(component + "(actual=" + actualCount + ", required=" +
- cardinality.cardinality + ")");
- }
- }
- return cardinalityFailures;
- }
-
- /**
- * Verify that a component is included in all host groups.
- * For components that are auto-install enabled, will add component to topology if needed.
- *
- * @param blueprint blueprint instance
- * @param hostGroups collection of host groups
- * @param component component to validate
- * @param autoDeploy auto-deploy information for component
- *
- * @return collection of missing component information
- */
- private Collection<String> verifyComponentInAllHostGroups(BlueprintEntity blueprint,
- Collection<HostGroupImpl> hostGroups,
- String component,
- AutoDeployInfo autoDeploy) {
-
- Collection<String> cardinalityFailures = new HashSet<String>();
- int actualCount = getHostGroupsForComponent(component, hostGroups).size();
- if (actualCount != hostGroups.size()) {
- if (autoDeploy != null && autoDeploy.isEnabled()) {
- for (HostGroupImpl group : hostGroups) {
- if (group.addComponent(component)) {
- addComponentToBlueprint(blueprint, group.getEntity().getName(), component);
- }
- }
- } else {
- cardinalityFailures.add(component + "(actual=" + actualCount + ", required=ALL)");
- }
- }
- return cardinalityFailures;
- }
-
- /**
- * Generate an exception for topology validation failure.
- *
- * @param missingDependencies missing dependency information
- * @param cardinalityFailures missing service component information
- *
- * @throws IllegalArgumentException Always thrown and contains information regarding the topology validation failure
- * in the msg
- */
- private void generateInvalidTopologyException(Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies,
- Collection<String> cardinalityFailures) {
-
- String msg = "Cluster Topology validation failed.";
- if (! cardinalityFailures.isEmpty()) {
- msg += " Invalid service component count: " + cardinalityFailures;
- }
- if (! missingDependencies.isEmpty()) {
- msg += " Unresolved component dependencies: " + missingDependencies;
- }
- msg += ". To disable topology validation and create the blueprint, " +
- "add the following to the end of the url: '?validate_topology=false'";
- throw new IllegalArgumentException(msg);
- }
-
- /**
- * Create host and host_component resources.
- *
- * @param blueprintHostGroups host groups specified in blueprint
- * @param clusterName cluster name
- *
- * @throws SystemException an unexpected exception occurred
- * @throws UnsupportedPropertyException an invalid property was specified
- * @throws ResourceAlreadyExistsException attempt to create a host or host_component which already exists
- * @throws NoSuchParentResourceException a required parent resource is missing
- */
- protected void createHostAndComponentResources(Map<String, HostGroupImpl> blueprintHostGroups, String clusterName)
- throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException {
-
- createHostAndComponentResources(blueprintHostGroups, clusterName, getResourceProvider(Resource.Type.Host));
- }
-
- /**
- * Create host and host_component resources via the specified host resource provider.
- *
- * @param blueprintHostGroups host groups specified in blueprint
- * @param clusterName cluster name
- * @param hostProvider host resource provider
- *
- * @throws SystemException an unexpected exception occurred
- * @throws UnsupportedPropertyException an invalid property was specified
- * @throws ResourceAlreadyExistsException attempt to create a host or host_component which already exists
- * @throws NoSuchParentResourceException a required parent resource is missing
- */
- protected void createHostAndComponentResources(Map<String, HostGroupImpl> blueprintHostGroups,
- String clusterName,
- ResourceProvider hostProvider)
- throws SystemException,
- UnsupportedPropertyException,
- ResourceAlreadyExistsException,
- NoSuchParentResourceException {
-
- ResourceProvider hostComponentProvider = getResourceProvider(Resource.Type.HostComponent);
- for (HostGroupImpl group : blueprintHostGroups.values()) {
- for (String host : group.getHostInfo()) {
- Map<String, Object> hostProperties = new HashMap<String, Object>();
- hostProperties.put("Hosts/cluster_name", clusterName);
- hostProperties.put("Hosts/host_name", host);
-
- hostProvider.createResources(new RequestImpl(
- null, Collections.singleton(hostProperties), null, null));
-
- // create clusters/hosts/host_components
- Set<Map<String, Object>> setHostComponentRequestProps = new HashSet<Map<String, Object>>();
- for (String hostComponent : group.getComponents()) {
- // AMBARI_SERVER is not recognized by Ambari as a component
- if (! hostComponent.equals("AMBARI_SERVER")) {
- Map<String, Object> hostComponentProperties = new HashMap<String, Object>();
- hostComponentProperties.put("HostRoles/cluster_name", clusterName);
- hostComponentProperties.put("HostRoles/host_name", host);
- hostComponentProperties.put("HostRoles/component_name", hostComponent);
- setHostComponentRequestProps.add(hostComponentProperties);
- }
- }
- hostComponentProvider.createResources(new RequestImpl(
- null, setHostComponentRequestProps, null, null));
- }
- }
- }
-
- /**
- * Get a config group name based on a bp and host group.
- *
- * @param bpName blueprint name
- * @param hostGroupName host group name
- * @return config group name
- */
- protected String getConfigurationGroupName(String bpName, String hostGroupName) {
- return String.format("%s:%s", bpName, hostGroupName);
- }
-
-
- // ----- Inner Classes -----------------------------------------------------
-
- /**
- * Host group representation.
- */
- protected static class HostGroupImpl implements HostGroup {
- /**
- * Host group entity
- */
- private HostGroupEntity hostGroup;
-
- /**
- * Components contained in the host group
- */
- private Collection<String> components = new HashSet<String>();
-
- /**
- * Hosts contained associated with the host group
- */
- private Collection<String> hosts = new HashSet<String>();
-
- /**
- * Map of service to components for the host group
- */
- private Map<String, Set<String>> componentsForService = new HashMap<String, Set<String>>();
-
- /**
- * Map of host group configurations.
- * Type -> Map<Key, Val>
- */
- private Map<String, Map<String, String>> configurations =
- new HashMap<String, Map<String, String>>();
-
- /**
- * Associated stack
- */
- private Stack stack;
-
- /**
- * The Blueprint processor associated with this HostGroupImpl instance
- */
- private final BaseBlueprintProcessor blueprintProcessor;
-
- /**
- * Constructor.
- *
- * @param hostGroup host group
- * @param stack stack
- */
- public HostGroupImpl(HostGroupEntity hostGroup, Stack stack, BaseBlueprintProcessor blueprintProcessor) {
- this.hostGroup = hostGroup;
- this.stack = stack;
- this.blueprintProcessor = blueprintProcessor;
- parseComponents();
- parseConfigurations();
- }
-
- @Override
- public String getName() {
- return hostGroup.getName();
- }
-
- @Override
- public Collection<String> getComponents() {
- return components;
- }
-
- @Override
- public Collection<String> getHostInfo() {
- return hosts;
- }
-
- /**
- * Associate a host with the host group.
- *
- * @param fqdn fully qualified domain name of the host being added
- */
- public void addHostInfo(String fqdn) {
- hosts.add(fqdn);
- }
-
- /**
- * Get the services which are deployed to this host group.
- *
- * @return collection of services which have components in this host group
- */
- public Collection<String> getServices() {
- return componentsForService.keySet();
- }
-
- /**
- * Add a component to the host group.
- *
- * @param component component to add
- *
- * @return true if component was added; false if component already existed
- */
- public boolean addComponent(String component) {
- boolean added = components.add(component);
- if (added) {
- String service = stack.getServiceForComponent(component);
- if (service != null) {
- // an example of a component without a service in the stack is AMBARI_SERVER
- Set<String> serviceComponents = componentsForService.get(service);
- if (serviceComponents == null) {
- serviceComponents = new HashSet<String>();
- componentsForService.put(service, serviceComponents);
- }
- serviceComponents.add(component);
- }
- }
- return added;
- }
-
- /**
- * Get the components for the specified service which are associated with the host group.
- *
- * @param service service name
- *
- * @return set of component names
- */
- public Collection<String> getComponents(String service) {
- return componentsForService.get(service);
- }
-
- /**
- * Get the configurations associated with the host group.
- *
- * @return map of configuration type to a map of properties
- */
- @Override
- public Map<String, Map<String, String>> getConfigurationProperties() {
- return configurations;
- }
-
- /**
- * Get the associated entity.
- *
- * @return associated host group entity
- */
- public HostGroupEntity getEntity() {
- return hostGroup;
- }
-
- /**
- * Validate host group topology. This includes ensuring that all component dependencies are satisfied.
- *
- * @param hostGroups collection of all host groups
- * @param services set of services in cluster topology
- * @param clusterConfig cluster configuration
- *
- * @return map of component to missing dependencies
- */
- public Map<String, Collection<DependencyInfo>> validateTopology(Collection<HostGroupImpl> hostGroups,
- Collection<String> services,
- Map<String, Map<String, String>> clusterConfig) {
-
- Map<String, Collection<DependencyInfo>> missingDependencies =
- new HashMap<String, Collection<DependencyInfo>>();
-
- for (String component : new HashSet<String>(components)) {
- Collection<DependencyInfo> dependenciesForComponent = stack.getDependenciesForComponent(component);
- for (DependencyInfo dependency : dependenciesForComponent) {
- String conditionalService = stack.getConditionalServiceForDependency(dependency);
- if (conditionalService != null && ! services.contains(conditionalService)) {
- continue;
- }
-
- BlueprintEntity entity = hostGroup.getBlueprintEntity();
- String dependencyScope = dependency.getScope();
- String componentName = dependency.getComponentName();
- AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy();
- boolean resolved = false;
-
- if (dependencyScope.equals("cluster")) {
- Collection<String> missingDependencyInfo = blueprintProcessor.verifyComponentCardinalityCount(entity, hostGroups,
- componentName, new Cardinality("1+"), autoDeployInfo, stack, clusterConfig);
- resolved = missingDependencyInfo.isEmpty();
- } else if (dependencyScope.equals("host")) {
- if (components.contains(component) || (autoDeployInfo != null && autoDeployInfo.isEnabled())) {
- resolved = true;
- if (addComponent(componentName)) {
- blueprintProcessor.addComponentToBlueprint(hostGroup.getBlueprintEntity(), getEntity().getName(), componentName);
- }
- }
- }
-
- if (! resolved) {
- Collection<DependencyInfo> missingCompDependencies = missingDependencies.get(component);
- if (missingCompDependencies == null) {
- missingCompDependencies = new HashSet<DependencyInfo>();
- missingDependencies.put(component, missingCompDependencies);
- }
- missingCompDependencies.add(dependency);
- }
- }
- }
- return missingDependencies;
- }
-
- /**
- * Parse component information.
- */
- private void parseComponents() {
- for (HostGroupComponentEntity componentEntity : hostGroup.getComponents() ) {
- addComponent(componentEntity.getName());
- }
- }
-
- /**
- * Parse host group configurations.
- */
- private void parseConfigurations() {
- Gson jsonSerializer = new Gson();
- for (HostGroupConfigEntity configEntity : hostGroup.getConfigurations()) {
- String type = configEntity.getType();
- Map<String, String> typeProperties = configurations.get(type);
- if ( typeProperties == null) {
- typeProperties = new HashMap<String, String>();
- configurations.put(type, typeProperties);
- }
- Map<String, String> propertyMap = jsonSerializer.<Map<String, String>>fromJson(
- configEntity.getConfigData(), Map.class);
-
- if (propertyMap != null) {
- typeProperties.putAll(propertyMap);
- }
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 9c3266a..95e9807 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -19,12 +19,20 @@
package org.apache.ambari.server.controller.internal;
+import org.apache.ambari.server.topology.Cardinality;
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.HostGroupInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
@@ -39,6 +47,8 @@ import java.util.regex.Pattern;
*/
public class BlueprintConfigurationProcessor {
+ protected final static Logger LOG = LoggerFactory.getLogger(BlueprintConfigurationProcessor.class);
+
/**
* Single host topology updaters
*/
@@ -95,33 +105,61 @@ public class BlueprintConfigurationProcessor {
private static Set<String> configPropertiesWithHASupport =
new HashSet<String>(Arrays.asList("fs.defaultFS", "hbase.rootdir", "instance.volumes"));
-
-
/**
* Configuration properties to be updated
*/
- private Map<String, Map<String, String>> properties;
+ //private Map<String, Map<String, String>> properties;
+ private ClusterTopology clusterTopology;
- /**
- * Constructor.
- *
- * @param properties properties to update
- */
- public BlueprintConfigurationProcessor(Map<String, Map<String, String>> properties) {
- this.properties = properties;
+
+ public BlueprintConfigurationProcessor(ClusterTopology clusterTopology) {
+ this.clusterTopology = clusterTopology;
+ }
+
+ public Collection<String> getRequiredHostGroups() {
+ Collection<String> requiredHostGroups = new HashSet<String>();
+
+ for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) {
+ for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) {
+ String type = entry.getKey();
+ for (Map.Entry<String, PropertyUpdater> updaterEntry : entry.getValue().entrySet()) {
+ String propertyName = updaterEntry.getKey();
+ PropertyUpdater updater = updaterEntry.getValue();
+
+ // topo cluster scoped configuration which also includes all default and BP properties
+ Map<String, Map<String, String>> clusterProps = clusterTopology.getConfiguration().getFullProperties();
+ Map<String, String> typeMap = clusterProps.get(type);
+ if (typeMap != null && typeMap.containsKey(propertyName)) {
+ requiredHostGroups.addAll(updater.getRequiredHostGroups(
+ typeMap.get(propertyName), clusterProps, clusterTopology));
+ }
+
+ // host group configs
+ for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
+ Map<String, Map<String, String>> hgConfigProps = groupInfo.getConfiguration().getProperties();
+ Map<String, String> hgTypeMap = hgConfigProps.get(type);
+ if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) {
+ requiredHostGroups.addAll(updater.getRequiredHostGroups(
+ hgTypeMap.get(propertyName), hgConfigProps, clusterTopology));
+ }
+ }
+ }
+ }
+ }
+ return requiredHostGroups;
}
+
/**
* Update properties for cluster creation. This involves updating topology related properties with
* concrete topology information.
- *
- * @param hostGroups host groups of cluster to be deployed
- * @param stackDefinition stack used for cluster creation
- *
- * @return updated properties
*/
- public Map<String, Map<String, String>> doUpdateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, Stack stackDefinition) {
+ public void doUpdateForClusterCreate() throws ConfigurationTopologyException {
+ Configuration clusterConfig = clusterTopology.getConfiguration();
+ Map<String, Map<String, String>> clusterProps = clusterConfig.getFullProperties();
+ Map<String, HostGroupInfo> groupInfoMap = clusterTopology.getHostGroupInfo();
+
for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) {
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) {
String type = entry.getKey();
@@ -129,45 +167,73 @@ public class BlueprintConfigurationProcessor {
String propertyName = updaterEntry.getKey();
PropertyUpdater updater = updaterEntry.getValue();
- Map<String, String> typeMap = properties.get(type);
+ // topo cluster scoped configuration which also includes all default and BP properties
+ Map<String, String> typeMap = clusterProps.get(type);
if (typeMap != null && typeMap.containsKey(propertyName)) {
- typeMap.put(propertyName, updater.updateForClusterCreate(
- hostGroups, propertyName, typeMap.get(propertyName), properties, stackDefinition));
+ clusterConfig.setProperty(type, propertyName, updater.updateForClusterCreate(
+ propertyName, typeMap.get(propertyName), clusterProps, clusterTopology));
+ }
+
+ // host group configs
+ for (HostGroupInfo groupInfo : groupInfoMap.values()) {
+ Configuration hgConfig = groupInfo.getConfiguration();
+ Map<String, Map<String, String>> hgConfigProps = hgConfig.getProperties();
+ Map<String, String> hgTypeMap = hgConfigProps.get(type);
+ if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) {
+ hgConfig.setProperty(type, propertyName, updater.updateForClusterCreate(
+ propertyName, hgTypeMap.get(propertyName), hgConfigProps, clusterTopology));
+ }
}
}
}
}
- if (isNameNodeHAEnabled()) {
+ //todo: lots of hard coded HA rules included here
+ if (clusterTopology.isNameNodeHAEnabled()) {
// if the active/stanbdy namenodes are not specified, assign them automatically
- if (! isNameNodeHAInitialActiveNodeSet(properties) && ! isNameNodeHAInitialStandbyNodeSet(properties)) {
- Collection<HostGroup> listOfHostGroups = new LinkedList<HostGroup>();
- for (String key : hostGroups.keySet()) {
- listOfHostGroups.add(hostGroups.get(key));
+ if (! isNameNodeHAInitialActiveNodeSet(clusterProps) && ! isNameNodeHAInitialStandbyNodeSet(clusterProps)) {
+ Collection<String> nnHosts = clusterTopology.getHostAssignmentsForComponent("NAMENODE");
+ if (nnHosts.size() != 2) {
+ throw new ConfigurationTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " +
+ nnHosts.size() + " Hosts: " + nnHosts);
}
- Collection<HostGroup> hostGroupsContainingNameNode =
- getHostGroupsForComponent("NAMENODE", listOfHostGroups);
// set the properties that configure which namenode is active,
// and which is a standby node in this HA deployment
- Map<String, String> hadoopEnv = properties.get("hadoop-env");
- if (hostGroupsContainingNameNode.size() == 2) {
- List<HostGroup> listOfGroups = new LinkedList<HostGroup>(hostGroupsContainingNameNode);
- hadoopEnv.put("dfs_ha_initial_namenode_active", listOfGroups.get(0).getHostInfo().iterator().next());
- hadoopEnv.put("dfs_ha_initial_namenode_standby", listOfGroups.get(1).getHostInfo().iterator().next());
- } else {
- // handle the case where multiple hosts are mapped to an HA host group
- if (hostGroupsContainingNameNode.size() == 1) {
- List<String> listOfInfo = new LinkedList<String>(hostGroupsContainingNameNode.iterator().next().getHostInfo());
- // there should only be two host names that can include a NameNode install/deployment
- hadoopEnv.put("dfs_ha_initial_namenode_active", listOfInfo.get(0));
- hadoopEnv.put("dfs_ha_initial_namenode_standby", listOfInfo.get(1));
- }
- }
+ Iterator<String> nnHostIterator = nnHosts.iterator();
+ clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_active", nnHostIterator.next());
+ clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", nnHostIterator.next());
}
}
+ setMissingConfigurations(clusterProps);
+ }
+
+ /**
+ * Update properties for blueprint export.
+ * This involves converting concrete topology information to host groups.
+ */
+ //todo: use cluster topology
+ public void doUpdateForBlueprintExport() {
+
+ // HA configs are only processed in cluster configuration, not HG configurations
+ if (clusterTopology.isNameNodeHAEnabled()) {
+ doNameNodeHAUpdate();
+ }
+ Collection<Map<String, Map<String, String>>> allConfigs = new ArrayList<Map<String, Map<String, String>>>();
+ allConfigs.add(clusterTopology.getConfiguration().getFullProperties());
+ for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
+ // don't use full properties, only the properties specified in the host group config
+ allConfigs.add(groupInfo.getConfiguration().getProperties());
+ }
+
+ for (Map<String, Map<String, String>> properties : allConfigs) {
+ doSingleHostExportUpdate(singleHostTopologyUpdaters, properties);
+ doSingleHostExportUpdate(dbHostTopologyUpdaters, properties);
- return properties;
+ doMultiHostExportUpdate(multiHostTopologyUpdaters, properties);
+
+ doRemovePropertyExport(removePropertyUpdaters, properties);
+ }
}
/**
@@ -179,7 +245,7 @@ public class BlueprintConfigurationProcessor {
* @return Collection of PropertyUpdater maps used to handle cluster config update
*/
private Collection<Map<String, Map<String, PropertyUpdater>>> createCollectionOfUpdaters() {
- return (isNameNodeHAEnabled()) ? addHAUpdaters(allUpdaters) : allUpdaters;
+ return (clusterTopology.isNameNodeHAEnabled()) ? addHAUpdaters(allUpdaters) : allUpdaters;
}
/**
@@ -208,29 +274,6 @@ public class BlueprintConfigurationProcessor {
}
/**
- * Update properties for blueprint export.
- * This involves converting concrete topology information to host groups.
- *
- * @param hostGroups cluster host groups
- *
- * @return updated properties
- */
- public Map<String, Map<String, String>> doUpdateForBlueprintExport(Collection<? extends HostGroup> hostGroups) {
- doSingleHostExportUpdate(hostGroups, singleHostTopologyUpdaters);
- doSingleHostExportUpdate(hostGroups, dbHostTopologyUpdaters);
-
- if (isNameNodeHAEnabled()) {
- doNameNodeHAUpdate(hostGroups);
- }
-
- doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters);
-
- doRemovePropertyExport(removePropertyUpdaters);
-
- return properties;
- }
-
- /**
* Performs export update for the set of properties that do not
* require update during cluster setup, but should be removed
* during a Blueprint export.
@@ -243,7 +286,9 @@ public class BlueprintConfigurationProcessor {
* @param updaters set of updaters for properties that should
* always be removed during a Blueprint export
*/
- private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters) {
+ private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters,
+ Map<String, Map<String, String>> properties) {
+
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
for (String propertyName : entry.getValue().keySet()) {
@@ -261,14 +306,13 @@ public class BlueprintConfigurationProcessor {
* dynamically determines the property names, and registers PropertyUpdaters to handle the masking of
* host names in these configuration items.
*
- * @param hostGroups cluster host groups
*/
- public void doNameNodeHAUpdate(Collection<? extends HostGroup> hostGroups) {
+ public void doNameNodeHAUpdate() {
Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfHAUpdaters();
// perform a single host update on these dynamically generated property names
if (highAvailabilityUpdaters.get("hdfs-site").size() > 0) {
- doSingleHostExportUpdate(hostGroups, highAvailabilityUpdaters);
+ doSingleHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties());
}
}
@@ -286,7 +330,8 @@ public class BlueprintConfigurationProcessor {
Map<String, PropertyUpdater> hdfsSiteUpdatersForAvailability = new HashMap<String, PropertyUpdater>();
highAvailabilityUpdaters.put("hdfs-site", hdfsSiteUpdatersForAvailability);
- Map<String, String> hdfsSiteConfig = properties.get("hdfs-site");
+ //todo: Do we need to call this for HG configurations?
+ Map<String, String> hdfsSiteConfig = clusterTopology.getConfiguration().getFullProperties().get("hdfs-site");
// generate the property names based on the current HA config for the NameNode deployments
for (String nameService : parseNameServices(hdfsSiteConfig)) {
for (String nameNode : parseNameNodes(nameService, hdfsSiteConfig)) {
@@ -302,27 +347,6 @@ public class BlueprintConfigurationProcessor {
}
/**
- * Convenience function to determine if NameNode HA is enabled.
- *
- * @return true if NameNode HA is enabled
- * false if NameNode HA is not enabled
- */
- boolean isNameNodeHAEnabled() {
- return isNameNodeHAEnabled(properties);
- }
-
- /**
- * Static convenience function to determine if NameNode HA is enabled
- * @param configProperties configuration properties for this cluster
- * @return true if NameNode HA is enabled
- * false if NameNode HA is not enabled
- */
- static boolean isNameNodeHAEnabled(Map<String, Map<String, String>> configProperties) {
- return configProperties.containsKey("hdfs-site") && configProperties.get("hdfs-site").containsKey("dfs.nameservices");
- }
-
-
- /**
* Static convenience function to determine if Yarn ResourceManager HA is enabled
* @param configProperties configuration properties for this cluster
* @return true if Yarn ResourceManager HA is enabled
@@ -413,11 +437,9 @@ public class BlueprintConfigurationProcessor {
/**
* Update single host topology configuration properties for blueprint export.
*
- * @param hostGroups cluster export
* @param updaters registered updaters
*/
- private void doSingleHostExportUpdate(Collection<? extends HostGroup> hostGroups,
- Map<String, Map<String, PropertyUpdater>> updaters) {
+ private void doSingleHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Map<String, Map<String, String>> properties) {
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
@@ -427,14 +449,15 @@ public class BlueprintConfigurationProcessor {
Map<String, String> typeProperties = properties.get(type);
if (typeProperties != null && typeProperties.containsKey(propertyName)) {
String propValue = typeProperties.get(propertyName);
- for (HostGroup group : hostGroups) {
- Collection<String> hosts = group.getHostInfo();
+
+ for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
+ Collection<String> hosts = groupInfo.getHostNames();
for (String host : hosts) {
//todo: need to use regular expression to avoid matching a host which is a superset.
if (propValue.contains(host)) {
matchedHost = true;
typeProperties.put(propertyName, propValue.replace(
- host, "%HOSTGROUP::" + group.getName() + "%"));
+ host, "%HOSTGROUP::" + groupInfo.getHostGroupName() + "%"));
break;
}
}
@@ -501,22 +524,20 @@ public class BlueprintConfigurationProcessor {
/**
* Update multi host topology configuration properties for blueprint export.
*
- * @param hostGroups cluster host groups
* @param updaters registered updaters
*/
- private void doMultiHostExportUpdate(Collection<? extends HostGroup> hostGroups,
- Map<String, Map<String, PropertyUpdater>> updaters) {
-
+ private void doMultiHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Map<String, Map<String, String>> properties) {
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
for (String propertyName : entry.getValue().keySet()) {
Map<String, String> typeProperties = properties.get(type);
if (typeProperties != null && typeProperties.containsKey(propertyName)) {
String propValue = typeProperties.get(propertyName);
- for (HostGroup group : hostGroups) {
- Collection<String> hosts = group.getHostInfo();
+ for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
+ Collection<String> hosts = groupInfo.getHostNames();
for (String host : hosts) {
- propValue = propValue.replaceAll(host + "\\b", "%HOSTGROUP::" + group.getName() + "%");
+ propValue = propValue.replaceAll(host + "\\b", "%HOSTGROUP::" +
+ groupInfo.getHostGroupName() + "%");
}
}
Collection<String> addedGroups = new HashSet<String>();
@@ -550,35 +571,16 @@ public class BlueprintConfigurationProcessor {
}
/**
- * Get host groups which contain a component.
- *
- * @param component component name
- * @param hostGroups collection of host groups to check
- *
- * @return collection of host groups which contain the specified component
- */
- private static Collection<HostGroup> getHostGroupsForComponent(String component,
- Collection<? extends HostGroup> hostGroups) {
-
- Collection<HostGroup> resultGroups = new LinkedHashSet<HostGroup>();
- for (HostGroup group : hostGroups ) {
- if (group.getComponents().contains(component)) {
- resultGroups.add(group);
- }
- }
- return resultGroups;
- }
-
- /**
* Convert a property value which includes a host group topology token to a physical host.
*
- * @param hostGroups cluster host groups
- * @param val value to be converted
+ *
+ * @param val value to be converted
+ * @param topology cluster topology
*
* @return updated value with physical host name
*/
- private static Collection<String> getHostStrings(Map<String, ? extends HostGroup> hostGroups,
- String val) {
+ //todo: replace this with parseHostGroupToken which would return a hostgroup or null
+ private static Collection<String> getHostStrings(String val, ClusterTopology topology) {
Collection<String> hosts = new LinkedHashSet<String>();
Matcher m = HOSTGROUP_PORT_REGEX.matcher(val);
@@ -586,13 +588,13 @@ public class BlueprintConfigurationProcessor {
String groupName = m.group(1);
String port = m.group(2);
-
- HostGroup hostGroup = hostGroups.get(groupName);
- if (hostGroup == null) {
+ HostGroupInfo hostGroupInfo = topology.getHostGroupInfo().get(groupName);
+ if (hostGroupInfo == null) {
throw new IllegalArgumentException(
"Unable to match blueprint host group token to a host group: " + groupName);
}
- for (String host : hostGroup.getHostInfo()) {
+
+ for (String host : hostGroupInfo.getHostNames()) {
if (port != null) {
host += ":" + port;
}
@@ -627,26 +629,28 @@ public class BlueprintConfigurationProcessor {
/**
* Update a property value.
*
- *
- * @param hostGroups host groups
- * @param propertyName name of property
+ * @param propertyName name of
* @param origValue original value of property
* @param properties all properties
- * @param stackDefinition definition of stack used for this cluster
- * creation attempt
+ * @param topology cluster topology
*
* @return new property value
*/
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
- String propertyName, String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition
- );
+ public String updateForClusterCreate(String propertyName,
+ String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology);
+
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology);
}
/**
* Topology based updater which replaces the original host name of a property with the host name
* which runs the associated (master) component in the new cluster.
*/
- static class SingleHostTopologyUpdater implements PropertyUpdater {
+ private static class SingleHostTopologyUpdater implements PropertyUpdater {
/**
* Component name
*/
@@ -664,43 +668,52 @@ public class BlueprintConfigurationProcessor {
/**
* Update the property with the new host name which runs the associated component.
*
- *
- * @param hostGroups host groups
- * @param propertyName name of property
- * @param origValue original value of property
- * @param properties all properties
- * @param stackDefinition stack used for cluster creation
+ * @param propertyName name of property
+ * @param origValue original value of property
+ * @param properties all properties
+ * @param topology cluster topology
*
* @return updated property value with old host name replaced by new host name
*/
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
- String propertyName,
+ public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- Stack stackDefinition) {
+ ClusterTopology topology) {
+ //todo: getHostStrings
Matcher m = HOSTGROUP_REGEX.matcher(origValue);
if (m.find()) {
String hostGroupName = m.group(1);
- HostGroup hostGroup = hostGroups.get(hostGroupName);
- //todo: ensure > 0 hosts (is this necessary)
- return origValue.replace(m.group(0), hostGroup.getHostInfo().iterator().next());
+
+ HostGroupInfo groupInfo = topology.getHostGroupInfo().get(hostGroupName);
+ if (groupInfo == null) {
+ //todo: this should be validated in configuration validation
+ throw new RuntimeException(
+ "Encountered a host group token in configuration which couldn't be matched to a host group: "
+ + hostGroupName);
+ }
+
+ //todo: warn if > hosts
+ return origValue.replace(m.group(0), groupInfo.getHostNames().iterator().next());
} else {
- Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values());
- if (matchingGroups.size() == 1) {
- return origValue.replace("localhost", matchingGroups.iterator().next().getHostInfo().iterator().next());
+ int matchingGroupCount = topology.getHostGroupsForComponent(component).size();
+ if (matchingGroupCount == 1) {
+ Collection<String> componentHosts = topology.getHostAssignmentsForComponent(component);
+ //todo: warn if > 1 hosts
+ return origValue.replace("localhost", componentHosts.iterator().next());
} else {
- Cardinality cardinality = stackDefinition.getCardinality(component);
+ //todo: extract all hard coded HA logic
+ Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component);
// if no matching host groups are found for a component whose configuration
// is handled by this updater, check the stack first to determine if
// zero is a valid cardinality for this component. This is necessary
// in the case of a component in "technical preview" status, since it
// may be valid to have 0 or 1 instances of such a component in the cluster
- if (matchingGroups.isEmpty() && cardinality.isValidCount(0)) {
+ if (matchingGroupCount == 0 && cardinality.isValidCount(0)) {
return origValue;
} else {
- if (isNameNodeHAEnabled(properties) && isComponentNameNode() && (matchingGroups.size() == 2)) {
+ if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroupCount == 2)) {
// if this is the defaultFS property, it should reflect the nameservice name,
// rather than a hostname (used in non-HA scenarios)
if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) {
@@ -726,34 +739,34 @@ public class BlueprintConfigurationProcessor {
}
- if (isNameNodeHAEnabled(properties) && isComponentSecondaryNameNode() && (matchingGroups.isEmpty())) {
+ if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroupCount == 0)) {
// if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE
// eventually this type of information should be encoded in the stacks
return origValue;
}
- if (isYarnResourceManagerHAEnabled(properties) && isComponentResourceManager() && (matchingGroups.size() == 2)) {
+ if (isYarnResourceManagerHAEnabled(properties) && isComponentResourceManager() && (matchingGroupCount == 2)) {
if (!origValue.contains("localhost")) {
// if this Yarn property is a FQDN, then simply return it
return origValue;
}
}
- if ((isOozieServerHAEnabled(properties)) && isComponentOozieServer() && (matchingGroups.size() > 1)) {
+ if ((isOozieServerHAEnabled(properties)) && isComponentOozieServer() && (matchingGroupCount > 1)) {
if (!origValue.contains("localhost")) {
// if this Oozie property is a FQDN, then simply return it
return origValue;
}
}
- if ((isHiveServerHAEnabled(properties)) && isComponentHiveServer() && (matchingGroups.size() > 1)) {
+ if ((isHiveServerHAEnabled(properties)) && isComponentHiveServer() && (matchingGroupCount > 1)) {
if (!origValue.contains("localhost")) {
// if this Hive property is a FQDN, then simply return it
return origValue;
}
}
- if ((isComponentHiveMetaStoreServer()) && matchingGroups.size() > 1) {
+ if ((isComponentHiveMetaStoreServer()) && matchingGroupCount > 1) {
if (!origValue.contains("localhost")) {
// if this Hive MetaStore property is a FQDN, then simply return it
return origValue;
@@ -768,6 +781,47 @@ public class BlueprintConfigurationProcessor {
}
}
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
+ //todo: getHostStrings
+ Matcher m = HOSTGROUP_REGEX.matcher(origValue);
+ if (m.find()) {
+ String hostGroupName = m.group(1);
+ return Collections.singleton(hostGroupName);
+ } else {
+ Collection<String> matchingGroups = topology.getHostGroupsForComponent(component);
+ if (matchingGroups.size() == 1) {
+ return Collections.singleton(matchingGroups.iterator().next());
+ } else {
+ if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroups.size() == 2)) {
+ // if this is the defaultFS property, it should reflect the nameservice name,
+ // rather than a hostname (used in non-HA scenarios)
+ if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) {
+ return Collections.emptySet();
+ }
+
+ if (properties.containsKey("hbase-site") && properties.get("hbase-site").get("hbase.rootdir").equals(origValue)) {
+ // hbase-site's reference to the namenode is handled differently in HA mode, since the
+ // reference must point to the logical nameservice, rather than an individual namenode
+ return Collections.emptySet();
+ }
+ }
+
+ if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroups.isEmpty())) {
+ // if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE
+ // eventually this type of information should be encoded in the stacks
+ return Collections.emptySet();
+ }
+
+ //todo:
+ throw new IllegalArgumentException("Unable to determine required host groups for component. " +
+ "Component '" + component + "' is not mapped to any host group or is mapped to multiple groups.");
+ }
+ }
+ }
+
/**
* Utility method to determine if the component associated with this updater
* instance is an HDFS NameNode
@@ -863,14 +917,29 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String propertyName, String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition) {
+ public String updateForClusterCreate(String propertyName,
+ String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
try {
- return super.updateForClusterCreate(hostGroups, propertyName, origValue, properties, stackDefinition);
+ return super.updateForClusterCreate(propertyName, origValue, properties, topology);
} catch (IllegalArgumentException illegalArgumentException) {
// return the original value, since the optional component is not available in this cluster
return origValue;
}
}
+
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
+
+ try {
+ return super.getRequiredHostGroups(origValue, properties, topology);
+ } catch (IllegalArgumentException e) {
+ return Collections.emptySet();
+ }
+ }
}
/**
@@ -908,23 +977,23 @@ public class BlueprintConfigurationProcessor {
* runs the associated component. If the database is external (non-managed), return the
* original value.
*
- *
- * @param hostGroups host groups
- * @param origValue original value of property
- * @param properties all properties
- * @param stackDefinition stack used for cluster creation
+ * @param propertyName property name
+ * @param origValue original value of property
+ * @param properties all properties
+ * @param topology cluster topology
*
* @return updated property value with old host name replaced by new host name or original value
* if the database is external
*/
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
- String propertyName,
- String origValue, Map<String, Map<String, String>> properties,
- Stack stackDefinition) {
+
+ public String updateForClusterCreate(String propertyName,
+ String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
if (isDatabaseManaged(properties)) {
- return super.updateForClusterCreate(hostGroups, propertyName, origValue, properties, stackDefinition);
+ return super.updateForClusterCreate(propertyName, origValue, properties, topology);
} else {
return origValue;
}
@@ -948,7 +1017,6 @@ public class BlueprintConfigurationProcessor {
*/
private static class MultipleHostTopologyUpdater implements PropertyUpdater {
-
private static final Character DEFAULT_SEPARATOR = ',';
/**
@@ -996,35 +1064,32 @@ public class BlueprintConfigurationProcessor {
* Update all host names included in the original property value with new host names which run the associated
* component.
*
- *
- * @param hostGroups host groups
- *
- * @param origValue original value of property
- * @param properties all properties
- * @param stackDefinition stack used for cluster creation
- *
+ * @param propertyName property name
+ * @param origValue original value of property
+ * @param properties all properties
+ * @param topology cluster topology
* @return updated property value with old host names replaced by new host names
*/
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
- String propertyName,
+ public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- Stack stackDefinition) {
+ ClusterTopology topology) {
+
StringBuilder sb = new StringBuilder();
if (!origValue.contains("%HOSTGROUP") &&
- (!origValue.contains("localhost"))) {
+ (!origValue.contains("localhost"))) {
// this property must contain FQDNs specified directly by the user
// of the Blueprint, so the processor should not attempt to update them
return origValue;
}
String prefix = null;
- Collection<String> hostStrings = getHostStrings(hostGroups, origValue);
+ Collection<String> hostStrings = getHostStrings(origValue, topology);
if (hostStrings.isEmpty()) {
//default non-exported original value
- String port = null;
+ String port;
for (String urlScheme : setOfKnownURLSchemes) {
if (origValue.startsWith(urlScheme)) {
prefix = urlScheme;
@@ -1039,22 +1104,15 @@ public class BlueprintConfigurationProcessor {
port = calculatePort(origValue);
}
-
- Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values());
- for (HostGroup group : matchingGroups) {
- for (String host : group.getHostInfo()) {
- if (port != null) {
- host += ":" + port;
- }
- hostStrings.add(host);
+ for (String host : topology.getHostAssignmentsForComponent(component)) {
+ if (port != null) {
+ host += ":" + port;
}
+ hostStrings.add(host);
}
}
-
-
String suffix = null;
-
// parse out prefix if one exists
Matcher matcher = HOSTGROUP_PORT_REGEX.matcher(origValue);
if (matcher.find()) {
@@ -1075,7 +1133,6 @@ public class BlueprintConfigurationProcessor {
if ((indexOfEnd > -1) && (indexOfEnd < (origValue.length() - 1))) {
suffix = origValue.substring(indexOfEnd);
}
-
}
// add hosts to property, using the specified separator
@@ -1090,17 +1147,12 @@ public class BlueprintConfigurationProcessor {
} else {
firstHost = false;
}
-
-
-
sb.append(host);
}
if ((suffix != null) && (!suffix.equals("']"))) {
sb.append(suffix);
}
-
-
return sb.toString();
}
@@ -1112,6 +1164,34 @@ public class BlueprintConfigurationProcessor {
return null;
}
+
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
+
+ Collection<String> requiredHostGroups = new HashSet<String>();
+
+ // add all host groups specified in host group tokens
+ Matcher m = HOSTGROUP_PORT_REGEX.matcher(origValue);
+ while (m.find()) {
+ String groupName = m.group(1);
+
+ if (!topology.getBlueprint().getHostGroups().containsKey(groupName)) {
+ throw new IllegalArgumentException(
+ "Unable to match blueprint host group token to a host group: " + groupName);
+ }
+ requiredHostGroups.add(groupName);
+ }
+
+ //todo: for now assuming that we will either have HG tokens or standard replacement but not both
+ //todo: as is done in updateForClusterCreate
+ if (requiredHostGroups.isEmpty()) {
+ requiredHostGroups.addAll(topology.getHostGroupsForComponent(component));
+ }
+
+ return requiredHostGroups;
+ }
}
/**
@@ -1122,23 +1202,27 @@ public class BlueprintConfigurationProcessor {
/**
* Append 'm' to the original property value if it doesn't already exist.
*
- *
- * @param hostGroups host groups
- * @param origValue original value of property
- * @param properties all properties
- * @param stackDefinition stack used for cluster creation
+ * @param propertyName property name
+ * @param origValue original value of property
+ * @param properties all properties
+ * @param topology cluster topology
*
* @return property with 'm' appended
*/
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
- String propertyName,
- String origValue, Map<String,
- Map<String, String>> properties,
- Stack stackDefinition) {
+ public String updateForClusterCreate(String propertyName,
+ String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
return origValue.endsWith("m") ? origValue : origValue + 'm';
}
+
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties, ClusterTopology topology) {
+ return Collections.emptySet();
+ }
}
/**
@@ -1159,21 +1243,20 @@ public class BlueprintConfigurationProcessor {
/**
* Return decorated form of the updated input property value.
*
- * @param hostGroupMap map of host group name to HostGroup
- * @param origValue original value of property
- * @param properties all properties
- * @param stackDefinition stack used for cluster creation
+ * @param propertyName property name
+ * @param origValue original value of property
+ * @param properties all properties
+ * @param topology cluster topology
*
* @return Formatted output string
*/
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroupMap,
- String propertyName,
+ public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- Stack stackDefinition) {
+ ClusterTopology topology) {
- return doFormat(propertyUpdater.updateForClusterCreate(hostGroupMap, propertyName, origValue, properties, stackDefinition));
+ return doFormat(propertyUpdater.updateForClusterCreate(propertyName, origValue, properties, topology));
}
/**
@@ -1184,6 +1267,13 @@ public class BlueprintConfigurationProcessor {
* @return formatted output string
*/
public abstract String doFormat(String originalValue);
+
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties, ClusterTopology topology) {
+
+ return propertyUpdater.getRequiredHostGroups(origValue, properties, topology);
+ }
}
/**
@@ -1232,14 +1322,22 @@ public class BlueprintConfigurationProcessor {
* during the Blueprint export process.
*/
private static class OriginalValuePropertyUpdater implements PropertyUpdater {
- @Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups,
- String propertyName, String origValue,
+
+ public String updateForClusterCreate(String propertyName,
+ String origValue,
Map<String, Map<String, String>> properties,
- Stack stackDefinition) {
+ ClusterTopology topology) {
// always return the original value, since these properties do not require update handling
return origValue;
}
+
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
+
+ return Collections.emptySet();
+ }
}
@@ -1264,7 +1362,11 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String propertyName, String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition) {
+ public String updateForClusterCreate(String propertyName,
+ String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
+
// short-circuit out any custom property values defined by the deployer
if (!origValue.contains("%HOSTGROUP") &&
(!origValue.contains("localhost"))) {
@@ -1273,8 +1375,7 @@ public class BlueprintConfigurationProcessor {
return origValue;
}
- StringBuffer updatedResult = new StringBuffer();
-
+ StringBuilder updatedResult = new StringBuilder();
// split out the key/value pairs
String[] keyValuePairs = origValue.split(",");
boolean firstValue = true;
@@ -1287,10 +1388,13 @@ public class BlueprintConfigurationProcessor {
String key = keyValuePair.split("=")[0];
if (mapOfKeysToUpdaters.containsKey(key)) {
- String result = mapOfKeysToUpdaters.get(key).updateForClusterCreate(hostGroups, key, keyValuePair.split("=")[1], properties, stackDefinition);
+ String result = mapOfKeysToUpdaters.get(key).updateForClusterCreate(
+ key, keyValuePair.split("=")[1], properties, topology);
// append the internal property result, escape out any commas in the internal property,
// this is required due to the specific syntax of templeton.hive.properties
- updatedResult.append(key + "=" + result.replaceAll(",", Matcher.quoteReplacement("\\,")));
+ updatedResult.append(key);
+ updatedResult.append("=");
+ updatedResult.append(result.replaceAll(",", Matcher.quoteReplacement("\\,")));
} else {
updatedResult.append(keyValuePair);
}
@@ -1298,6 +1402,32 @@ public class BlueprintConfigurationProcessor {
return updatedResult.toString();
}
+
+ @Override
+ public Collection<String> getRequiredHostGroups(String origValue,
+ Map<String, Map<String, String>> properties,
+ ClusterTopology topology) {
+
+ // short-circuit out any custom property values defined by the deployer
+ if (!origValue.contains("%HOSTGROUP") &&
+ (!origValue.contains("localhost"))) {
+ // this property must contain FQDNs specified directly by the user
+ // of the Blueprint, so the processor should not attempt to update them
+ return Collections.emptySet();
+ }
+
+ Collection<String> requiredGroups = new HashSet<String>();
+ // split out the key/value pairs
+ String[] keyValuePairs = origValue.split(",");
+ for (String keyValuePair : keyValuePairs) {
+ String key = keyValuePair.split("=")[0];
+ if (mapOfKeysToUpdaters.containsKey(key)) {
+ requiredGroups.addAll(mapOfKeysToUpdaters.get(key).getRequiredHostGroups(
+ keyValuePair.split("=")[1], properties, topology));
+ }
+ }
+ return requiredGroups;
+ }
}
/**
@@ -1377,6 +1507,10 @@ public class BlueprintConfigurationProcessor {
removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap);
removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap);
+ //todo: Need to change updaters back to being static
+ //todo: will need to pass ClusterTopology in as necessary
+
+
// NAMENODE
hdfsSiteMap.put("dfs.http.address", new SingleHostTopologyUpdater("NAMENODE"));
hdfsSiteMap.put("dfs.https.address", new SingleHostTopologyUpdater("NAMENODE"));
@@ -1489,4 +1623,72 @@ public class BlueprintConfigurationProcessor {
hbaseEnvMap.put("hbase_master_heapsize", new MPropertyUpdater());
hbaseEnvMap.put("hbase_regionserver_heapsize", new MPropertyUpdater());
}
+
+ /**
+ * Explicitly set any properties that are required but not currently provided in the stack definition.
+ */
+ void setMissingConfigurations(Map<String, Map<String, String>> mapClusterConfigurations) {
+ // AMBARI-5206
+ final Map<String , String> userProps = new HashMap<String , String>();
+
+ Collection<String> services = clusterTopology.getBlueprint().getServices();
+ // only add user properties to the map for
+ // services actually included in the blueprint definition
+ if (services.contains("OOZIE")) {
+ userProps.put("oozie_user", "oozie-env");
+ }
+
+ if (services.contains("HIVE")) {
+ userProps.put("hive_user", "hive-env");
+ userProps.put("hcat_user", "hive-env");
+ }
+
+ if (services.contains("HBASE")) {
+ userProps.put("hbase_user", "hbase-env");
+ }
+
+ if (services.contains("FALCON")) {
+ userProps.put("falcon_user", "falcon-env");
+ }
+
+
+ String proxyUserHosts = "hadoop.proxyuser.%s.hosts";
+ String proxyUserGroups = "hadoop.proxyuser.%s.groups";
+
+ for (String property : userProps.keySet()) {
+ String configType = userProps.get(property);
+ Map<String, String> configs = mapClusterConfigurations.get(configType);
+ if (configs != null) {
+ String user = configs.get(property);
+ if (user != null && !user.isEmpty()) {
+ ensureProperty(mapClusterConfigurations, "core-site", String.format(proxyUserHosts, user), "*");
+ ensureProperty(mapClusterConfigurations, "core-site", String.format(proxyUserGroups, user), "users");
+ }
+ } else {
+ LOG.debug("setMissingConfigurations: no user configuration found for type = " + configType +
+ ". This may be caused by an error in the blueprint configuration.");
+ }
+
+ }
+ }
+
+ /**
+ * Ensure that the specified property exists.
+ * If not, set a default value.
+ *
+ * @param type config type
+ * @param property property name
+ * @param defaultValue default value
+ */
+ private void ensureProperty(Map<String, Map<String, String>> mapClusterConfigurations, String type, String property, String defaultValue) {
+ Map<String, String> properties = mapClusterConfigurations.get(type);
+ if (properties == null) {
+ properties = new HashMap<String, String>();
+ mapClusterConfigurations.put(type, properties);
+ }
+
+ if (! properties.containsKey(property)) {
+ properties.put(property, defaultValue);
+ }
+ }
}