You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by be...@apache.org on 2018/01/29 20:07:43 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: AMBARI-22791
support for parsing MulitE blueprints and cluster templates (#213)
This is an automated email from the ASF dual-hosted git repository.
benyoka pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 995b5d4 AMBARI-22791 support for parsing MulitE blueprints and cluster templates (#213)
995b5d4 is described below
commit 995b5d4251bf76963d6793a5fd58a4643b352761
Author: benyoka <be...@users.noreply.github.com>
AuthorDate: Mon Jan 29 21:07:40 2018 +0100
AMBARI-22791 support for parsing MulitE blueprints and cluster templates (#213)
* AMBARI-22776 Blueprint 3.0 Database schemas and JPA objects, DDL fixes (benyoka)
* AMBARI-22776 Revert accidental commit (benyoka)
* AMBARI-22776 Revert accidental change #2 (benyoka)
* AMBARI-22776 Fix review findings (benyoka)
* AMBARI-22776 Fix review findings #2 (benyoka)
* AMBARI-22791-Update blueprint object and convert from entity (benyoka)
* AMBARI-22791-BlueprintFactory and tests (benyoka)
* AMBARI-22791 Can store and retrieve multi-instance blueprint
* AMBARI-22791 Build and unit test fixes (benyoka)
* AMBARI-22791 merge branch-feature-AMBARI-14714 and fixes (benyoka)
* AMBARI-22791 fix unit test failure (benyoka)
* AMBARI-22791 fix import error (benyoka)
* AMBARI-22791 Multi-everythin cluster template support (benyoka)
* AMBARI-22791 fix review findings (benyoka)
---
.../api/query/render/ClusterBlueprintRenderer.java | 5 +-
.../StackAdvisorBlueprintProcessor.java | 2 +-
.../ambari/server/controller/AmbariServer.java | 3 +-
.../internal/BlueprintResourceProvider.java | 85 ++++--
.../internal/ClusterResourceProvider.java | 2 +
.../internal/ExportBlueprintRequest.java | 4 +-
.../internal/ProvisionClusterRequest.java | 47 ++++
.../ambari/server/controller/internal/Stack.java | 1 +
.../server/orm/entities/BlueprintConfigEntity.java | 1 -
.../orm/entities/BlueprintConfiguration.java | 7 -
.../server/orm/entities/BlueprintEntity.java | 2 +-
.../orm/entities/BlueprintMpackConfigEntity.java | 8 -
.../orm/entities/BlueprintServiceConfigEntity.java | 8 -
.../server/orm/entities/HostGroupConfigEntity.java | 2 -
.../ambari/server/stack/NoSuchStackException.java | 5 +
.../ambari/server/topology/AmbariContext.java | 2 +-
.../apache/ambari/server/topology/Blueprint.java | 39 ++-
.../ambari/server/topology/BlueprintFactory.java | 93 +++++--
.../ambari/server/topology/BlueprintImpl.java | 296 ++++++++++++++-------
.../server/topology/BlueprintValidatorImpl.java | 76 +++---
.../topology/ClusterConfigurationRequest.java | 6 +-
.../apache/ambari/server/topology/Component.java | 57 +++-
.../ambari/server/topology/Configurable.java | 93 +++++++
.../ambari/server/topology/Configuration.java | 6 +
.../apache/ambari/server/topology/HostGroup.java | 39 ++-
.../ambari/server/topology/HostGroupImpl.java | 201 +++++++++-----
.../ambari/server/topology/HostGroupInfo.java | 10 +
.../server/topology/ManagementPackMapping.java | 67 +++++
.../ambari/server/topology/MpackInstance.java | 159 +++++++++++
.../ambari/server/topology/ServiceInstance.java | 72 +++++
.../ambari/server/topology/TopologyManager.java | 4 +-
.../topology/tasks/PersistHostResourcesTask.java | 2 +-
.../topology/validators/HiveServiceValidator.java | 2 +-
.../src/main/resources/Ambari-DDL-MySQL-CREATE.sql | 1 -
ambari-server/src/main/resources/properties.json | 3 +-
.../BlueprintConfigurationProcessorTest.java | 98 ++++---
.../internal/BlueprintResourceProviderTest.java | 2 +-
.../internal/ExportBlueprintRequestTest.java | 16 +-
.../server/orm/entities/BlueprintEntityTest2.java | 3 -
.../security/encryption/CredentialStoreTest.java | 2 +-
.../ambari/server/topology/AmbariContextTest.java | 4 +-
.../server/topology/BlueprintFactoryTest.java | 91 ++++++-
.../ambari/server/topology/BlueprintImplTest.java | 26 +-
.../ambari/server/topology/BlueprintTestUtil.java | 79 ++++++
.../topology/BlueprintValidatorImplTest.java | 12 +-
.../topology/ClusterConfigurationRequestTest.java | 12 +-
.../topology/ClusterDeployWithStartOnlyTest.java | 12 +-
...terInstallWithoutStartOnComponentLevelTest.java | 35 ++-
.../topology/ClusterInstallWithoutStartTest.java | 12 +-
.../ambari/server/topology/ConfigurableTest.java | 97 +++++++
.../topology/RequiredPasswordValidatorTest.java | 12 +-
.../server/topology/TopologyManagerTest.java | 16 +-
.../validators/HiveServiceValidatorTest.java | 4 +-
.../test/resources/blueprint3.0/configurable.json | 16 ++
.../test/resources/blueprint3.0/configurable2.json | 8 +
.../blueprint3.0/multi-instance-blueprint.json | 104 ++++++++
56 files changed, 1667 insertions(+), 404 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index acdf9ed..6be0ff2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -59,7 +59,6 @@ import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.topology.AmbariContext;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.ClusterTopologyImpl;
-import org.apache.ambari.server.topology.Component;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.HostGroup;
import org.apache.ambari.server.topology.HostGroupInfo;
@@ -445,10 +444,10 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
*/
private List<Map<String, String>> processHostGroupComponents(HostGroup group) {
List<Map<String, String>> listHostGroupComponents = new ArrayList<>();
- for (Component component : group.getComponents()) {
+ for (String component : group.getComponentNames()) {
Map<String, String> mapComponentProperties = new HashMap<>();
listHostGroupComponents.add(mapComponentProperties);
- mapComponentProperties.put("name", component.getName());
+ mapComponentProperties.put("name", component);
}
return listHostGroupComponents;
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
index 273c0ff..ddd746b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
@@ -89,7 +89,7 @@ public class StackAdvisorBlueprintProcessor {
}
private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTopology, StackAdvisorRequestType requestType) {
- Stack stack = clusterTopology.getBlueprint().getStack();
+ Stack stack = clusterTopology.getBlueprint().getStack(); // TODO: implement multi-stack
Map<String, Set<String>> hgComponentsMap = gatherHostGroupComponents(clusterTopology);
Map<String, Set<String>> hgHostsMap = gatherHostGroupBindings(clusterTopology);
Map<String, Set<String>> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap,
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 9de89ee..5bad0f2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -941,8 +941,7 @@ public class AmbariServer {
StackDefinedPropertyProvider.init(injector);
AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class));
BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class),
- injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class),
- injector.getInstance(Gson.class), ambariMetaInfo);
+ injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class), ambariMetaInfo);
StackDependencyResourceProvider.init(ambariMetaInfo);
ClusterResourceProvider.init(injector.getInstance(TopologyManager.class),
injector.getInstance(TopologyRequestFactoryImpl.class), injector.getInstance(SecurityConfigurationFactory
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
index 7eef2df..1a4221e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
@@ -18,7 +18,12 @@
package org.apache.ambari.server.controller.internal;
+import static java.util.stream.Collectors.toList;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -57,17 +62,18 @@ import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.BlueprintFactory;
import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException;
import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.MpackInstance;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.apache.ambari.server.topology.SecurityConfigurationFactory;
import org.apache.ambari.server.utils.SecretReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
-import com.google.gson.Gson;
/**
@@ -101,6 +107,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
public static final String COMPONENT_PROPERTY_ID ="components";
public static final String COMPONENT_NAME_PROPERTY_ID ="name";
public static final String COMPONENT_PROVISION_ACTION_PROPERTY_ID = "provision_action";
+ protected static final String COMPONENT_MPACK_INSTANCE_PROPERTY = "mpack_instance";
+ protected static final String COMPONENT_SERVICE_INSTANCE_PROPERTY = "service_instance";
// Configurations
public static final String CONFIGURATION_PROPERTY_ID = "configurations";
@@ -120,6 +128,12 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
"Configuration elements must be Maps";
public static final String CONFIGURATION_MAP_SIZE_CHECK_ERROR_MESSAGE =
"Configuration Maps must hold a single configuration type each";
+ public static final String MPACK_INSTANCES_PROPERTY_ID = "mpack_instances";
+
+ // Primary Key Fields
+ private static Set<String> pkPropertyIds =
+ new HashSet<>(Arrays.asList(new String[]{
+ BLUEPRINT_NAME_PROPERTY_ID}));
/**
* The key property ids for a Blueprint resource.
@@ -138,7 +152,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
BLUEPRINT_SECURITY_PROPERTY_ID,
HOST_GROUP_PROPERTY_ID,
CONFIGURATION_PROPERTY_ID,
- SETTING_PROPERTY_ID);
+ SETTING_PROPERTY_ID,
+ MPACK_INSTANCES_PROPERTY_ID);
/**
* Used to create Blueprint instances
@@ -158,7 +173,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
/**
* Used to serialize to/from json.
*/
- private static Gson jsonSerializer;
+ private static ObjectMapper jsonSerializer = new ObjectMapper();
// ----- Constructors ----------------------------------------------------
@@ -176,14 +191,12 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
*
* @param factory blueprint factory
* @param dao blueprint data access object
- * @param gson json serializer
*/
public static void init(BlueprintFactory factory, BlueprintDAO dao, SecurityConfigurationFactory
- securityFactory, Gson gson, AmbariMetaInfo metaInfo) {
+ securityFactory, AmbariMetaInfo metaInfo) {
blueprintFactory = factory;
blueprintDAO = dao;
securityConfigurationFactory = securityFactory;
- jsonSerializer = gson;
ambariMetaInfo = metaInfo;
}
@@ -325,10 +338,16 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
for (HostGroupComponentEntity component : components) {
Map<String, String> mapComponentProps = new HashMap<>();
mapComponentProps.put(COMPONENT_NAME_PROPERTY_ID, component.getName());
-
if (component.getProvisionAction() != null) {
mapComponentProps.put(COMPONENT_PROVISION_ACTION_PROPERTY_ID, component.getProvisionAction().toString());
}
+ if (component.getMpackName() != null) {
+ mapComponentProps.put(COMPONENT_MPACK_INSTANCE_PROPERTY,
+ component.getMpackName() + "-" + component.getMpackVersion());
+ }
+ if (component.getServiceName() != null) {
+ mapComponentProps.put(COMPONENT_SERVICE_INSTANCE_PROPERTY, component.getServiceName());
+ }
listComponentProps.add(mapComponentProps);
}
@@ -351,9 +370,37 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
setResourceProperty(resource, BLUEPRINT_SECURITY_PROPERTY_ID, securityConfigMap, requestedIds);
}
+ Collection<Map<String, Object>> mpacks = entity.getMpackInstances().stream().map(mpackEntity -> {
+ MpackInstance mpack = MpackInstance.fromEntity(mpackEntity);
+ Map<String, Object> mpackAsMap = fromJson(toJson(mpack), Map.class);
+ return mpackAsMap;
+ } ).collect(toList());
+ setResourceProperty(resource, MPACK_INSTANCES_PROPERTY_ID, mpacks, requestedIds);
+
return resource;
}
+ private static <T> T fromJson(String json, Class<? extends T> valueType) {
+ if (null == json) {
+ return null;
+ }
+ try {
+ return jsonSerializer.readValue(json, valueType);
+ }
+ catch (IOException ex) {
+ throw new UncheckedIOException(ex);
+ }
+ }
+
+ private static String toJson(Object object) {
+ try {
+ return jsonSerializer.writeValueAsString(object);
+ }
+ catch (IOException ex) {
+ throw new UncheckedIOException(ex);
+ }
+ }
+
/**
* Populate a list of configuration property maps from a collection of configuration entities.
*
@@ -371,9 +418,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
String type = config.getType();
if(config instanceof BlueprintConfigEntity) {
- Map<String, String> properties = jsonSerializer.<Map<String, String>>fromJson(
- config.getConfigData(), Map.class);
-
+ Map<String, String> properties = fromJson(config.getConfigData(), Map.class);
// TODO: use multiple mpacks
BlueprintMpackInstanceEntity mpack =
@@ -387,19 +432,17 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
}
Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes =
- metaInfoStack.getConfigPropertiesTypes(type);
+ metaInfoStack.getConfigPropertiesTypes(type);
SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1l);
configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
} else {
- Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson(
- config.getConfigData(), Map.class);
+ Map<String, Object> properties = fromJson(config.getConfigData(), Map.class);
configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
}
- Map<String, Map<String, String>> attributes = jsonSerializer.<Map<String, Map<String, String>>>fromJson(
- config.getConfigAttributes(), Map.class);
+ Map<String, Map<String, String>> attributes = fromJson(config.getConfigAttributes(), Map.class);
if (attributes != null && !attributes.isEmpty()) {
configTypeDefinition.put(PROPERTIES_ATTRIBUTES_PROPERTY_ID, attributes);
}
@@ -423,8 +466,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
if (settings != null) {
for (BlueprintSettingEntity setting : settings) {
- List<Map<String, String>> propertiesList = jsonSerializer.<List<Map<String, String>>>fromJson(
- setting.getSettingData(), List.class);
+ List<Map<String, String>> propertiesList = fromJson(setting.getSettingData(), List.class);
Map<String, Object> settingMap = new HashMap<>();
settingMap.put(setting.getSettingName(), propertiesList);
listSettings.add(settingMap);
@@ -503,7 +545,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
String rawRequestBody = requestInfoProps.get(Request.REQUEST_INFO_BODY_PROPERTY);
Preconditions.checkArgument(!Strings.isNullOrEmpty(rawRequestBody), REQUEST_BODY_EMPTY_ERROR_MESSAGE);
- Map<String, Object> rawBodyMap = jsonSerializer.<Map<String, Object>>fromJson(rawRequestBody, Map.class);
+ Map<String, Object> rawBodyMap = fromJson(rawRequestBody, Map.class);
Object configurationData = rawBodyMap.get(CONFIGURATION_PROPERTY_ID);
if (configurationData != null) {
@@ -545,8 +587,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
}
LOG.info("Creating Blueprint, name=" + blueprint.getName());
- String blueprintSetting = blueprint.getSetting() == null ? "(null)" :
- jsonSerializer.toJson(blueprint.getSetting().getProperties());
+ String blueprintSetting = blueprint.getSetting() == null ? "(null)" : toJson(blueprint.getSetting().getProperties());
LOG.info("Blueprint setting=" + blueprintSetting);
try {
@@ -588,8 +629,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
}
}
- blueprintConfiguration.setConfigData(jsonSerializer.toJson(configData));
- blueprintConfiguration.setConfigAttributes(jsonSerializer.toJson(configAttributes));
+ blueprintConfiguration.setConfigData(toJson(configData));
+ blueprintConfiguration.setConfigAttributes(toJson(configAttributes));
}
protected abstract void addProperty(Map<String, String> configData,
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index 22960e1..7a1f517 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -86,6 +86,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
public static final String HEALTH_REPORT = "health_report";
public static final String CREDENTIAL_STORE_PROPERTIES = "credential_store_properties";
public static final String REPO_VERSION = "repository_version";
+ public static final String MPACK_INSTANCES = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID;
public static final String CLUSTER_ID_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_ID;
public static final String CLUSTER_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_NAME;
public static final String CLUSTER_VERSION_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + VERSION;
@@ -163,6 +164,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
propertyIds.add(SECURITY);
propertyIds.add(CREDENTIALS);
propertyIds.add(QUICKLINKS_PROFILE);
+ propertyIds.add(MPACK_INSTANCES);
propertyIds.add(CLUSTER_STATE_PROPERTY_ID);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
index 16d3114..ed6b5fd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
@@ -132,10 +132,10 @@ public class ExportBlueprintRequest implements TopologyRequest {
componentList.add(new Component(component));
}
- hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, stack, componentList,
+ hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, Collections.singleton(stack), componentList,
exportedHostGroup.getConfiguration(), String.valueOf(exportedHostGroup.getCardinality())));
}
- blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null);
+ blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null, null);
}
private void createHostGroupInfo(Collection<ExportedHostGroup> exportedHostGroups) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
index 1fd6091..115973c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
@@ -17,6 +17,9 @@
*/
package org.apache.ambari.server.controller.internal;
+import static java.util.stream.Collectors.toList;
+
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
@@ -34,11 +37,16 @@ import org.apache.ambari.server.topology.ConfigurationFactory;
import org.apache.ambari.server.topology.Credential;
import org.apache.ambari.server.topology.HostGroupInfo;
import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
+import org.apache.ambari.server.topology.ManagementPackMapping;
+import org.apache.ambari.server.topology.MpackInstance;
import org.apache.ambari.server.topology.NoSuchBlueprintException;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Enums;
import com.google.common.base.Optional;
import com.google.common.base.Strings;
@@ -119,6 +127,13 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
*/
public static final String QUICKLINKS_PROFILE_SERVICES_PROPERTY = "quicklinks_profile/services";
+ public static final String MANAGEMENT_PACK_MAPPINGS_PROPERTY = "management_pack_mappings";
+
+ public static final String MPACK_INSTANCES_PROPERTY = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID;
+
+ public static final String MPACK_INSTANCE_PROPERTY = "mpack_instance";
+
+ public static final String COMPONENT_NAME_PROPERTY = "component_name";
/**
* configuration factory
@@ -148,6 +163,8 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
private final String quickLinksProfileJson;
+ private Collection<MpackInstance> mpackInstances;
+
private final static Logger LOG = LoggerFactory.getLogger(ProvisionClusterRequest.class);
/**
@@ -196,6 +213,8 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
setProvisionAction(parseProvisionAction(properties));
+ processMpackInstances(properties);
+
try {
this.quickLinksProfileJson = processQuickLinksProfile(properties);
} catch (QuickLinksProfileEvaluationException ex) {
@@ -203,6 +222,21 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
}
}
+ private void processMpackInstances(Map<String, Object> properties) throws InvalidTopologyTemplateException {
+ if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+ try {
+ String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY));
+ this.mpackInstances = mapper.readValue(mpackInstancesJson,
+ new TypeReference<Collection<MpackInstance>>() {});
+ }
+ catch (IOException ex) {
+ throw new InvalidTopologyTemplateException("Cannot process mpack instances.", ex);
+ }
+ }
+ }
+
private String processQuickLinksProfile(Map<String, Object> properties) throws QuickLinksProfileEvaluationException {
Object globalFilters = properties.get(QUICKLINKS_PROFILE_FILTERS_PROPERTY);
Object serviceFilters = properties.get(QUICKLINKS_PROFILE_SERVICES_PROPERTY);
@@ -339,6 +373,16 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
processGroupHosts(name, (Collection<Map<String, String>>)
hostGroupProperties.get(HOSTGROUP_HOSTS_PROPERTY), hostGroupInfo);
+ // process mpack mappings
+ if (hostGroupProperties.containsKey(MANAGEMENT_PACK_MAPPINGS_PROPERTY)) {
+ Set<Map<String, String>> mpackMappingsRaw =
+ (Set<Map<String, String>>)hostGroupProperties.get(MANAGEMENT_PACK_MAPPINGS_PROPERTY);
+ Collection<ManagementPackMapping> mpackMappings = mpackMappingsRaw.stream().
+ map(mapping -> new ManagementPackMapping(mapping.get(COMPONENT_NAME_PROPERTY), mapping.get(MPACK_INSTANCE_PROPERTY))).
+ collect(toList());
+ hostGroupInfo.setManagementPackMappings(mpackMappings);
+ }
+
// don't set the parent configuration
hostGroupInfo.setConfiguration(configurationFactory.getConfiguration(
(Collection<Map<String, String>>) hostGroupProperties.get(CONFIGURATIONS_PROPERTY)));
@@ -490,4 +534,7 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
return defaultPassword;
}
+ public Collection<MpackInstance> getMpackInstances() {
+ return mpackInstances;
+ }
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
index a8fff2e..b54dd20 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
@@ -659,6 +659,7 @@ public class Stack {
if (componentDependencies != null && ! componentDependencies.isEmpty()) {
dependencies.put(componentName, componentDependencies);
}
+
if (component.isMaster()) {
masterComponents.add(componentName);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java
index 1620779..6a8e131 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java
@@ -58,7 +58,6 @@ public class BlueprintConfigEntity implements BlueprintConfiguration {
@JoinColumn(name = "blueprint_name", referencedColumnName = "blueprint_name", nullable = false)
private BlueprintEntity blueprint;
-
/**
* Get the configuration type.
*
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java
index cddac9b..186738b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java
@@ -37,13 +37,6 @@ public interface BlueprintConfiguration {
String getType();
/**
- * Set the blueprint name.
- *
- * @param blueprintName blueprint name
- */
- void setBlueprintName(String blueprintName);
-
- /**
* Get the blueprint name.
*
* @return blueprint name
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java
index ea92a7b..033cbd7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java
@@ -57,7 +57,7 @@ public class BlueprintEntity {
@Basic
@Column(name = "security_descriptor_reference", nullable = true, insertable = true, updatable = true)
private String securityDescriptorReference;
-
+
@OneToMany(cascade = CascadeType.ALL, mappedBy = "blueprint")
private Collection<HostGroupEntity> hostGroups = new ArrayList<>();
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java
index abc76a7..c1848d1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java
@@ -97,14 +97,6 @@ public class BlueprintMpackConfigEntity implements BlueprintConfiguration {
}
/**
- * @param blueprintName the name of the blueprint
- */
- @Override
- public void setBlueprintName(String blueprintName) {
- throw new UnsupportedOperationException();
- }
-
- /**
* @return the name of the blueprint
*/
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java
index a2bcf4e..d382288 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java
@@ -81,14 +81,6 @@ public class BlueprintServiceConfigEntity implements BlueprintConfiguration {
}
/**
- * @param blueprintName the blueprint name
- */
- @Override
- public void setBlueprintName(String blueprintName) {
- throw new UnsupportedOperationException("BlueprintServiceEntity.setBlueprintName");
- }
-
- /**
* @return the blueprint name
*/
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java
index dc64712..fb2f692 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java
@@ -60,7 +60,6 @@ public class HostGroupConfigEntity implements BlueprintConfiguration {
@Lob
private String configAttributes;
-
@ManyToOne
@JoinColumns({
@JoinColumn(name = "hostgroup_name", referencedColumnName = "name", nullable = false),
@@ -68,7 +67,6 @@ public class HostGroupConfigEntity implements BlueprintConfiguration {
})
private HostGroupEntity hostGroup;
-
/**
* Get the configuration type.
*
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java
index ba123de..47000db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java
@@ -25,4 +25,9 @@ public class NoSuchStackException extends Exception {
public NoSuchStackException(String stackName, String stackVersion) {
super(String.format("The requested stack doesn't exist. Name='%s' Version='%s'", stackName, stackVersion));
}
+
+ public NoSuchStackException(String stackName, String stackVersion, Throwable cause) {
+ super(String.format("The requested stack doesn't exist. Name='%s' Version='%s'", stackName, stackVersion), cause);
+ }
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 9f52972..37840f6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -350,7 +350,7 @@ public class AmbariContext {
serviceRequests.add(new ServiceRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, service,
repositoryVersionId, null, credentialStoreEnabled, null));
- for (String component : topology.getBlueprint().getComponents(service)) {
+ for (String component : topology.getBlueprint().getComponentNames(service)) {
String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component);
componentRequests.add(new ServiceComponentRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, null, recoveryEnabled));
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
index 6ed38f8..64d2ae3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
@@ -83,9 +83,19 @@ public interface Blueprint {
*
* @param service service name
*
+ * @return collection of components for the service. Will not return null.
+ */
+ Collection<Component> getComponents(String service);
+
+ /**
+ * Get the components that are included in the blueprint for the specified service.
+ *
+ * @param service service name
+ *
* @return collection of component names for the service. Will not return null.
*/
- Collection<String> getComponents(String service);
+ @Deprecated
+ Collection<String> getComponentNames(String service);
/**
* Get whether a component is enabled for auto start.
@@ -117,9 +127,25 @@ public interface Blueprint {
*
* @return associated stack
*/
+ @Deprecated
Stack getStack();
/**
+ * Get the stacks associated with the blueprint.
+ *
+ * @return associated stacks
+ */
+ Collection<Stack> getStacks();
+
+
+ /**
+ * Get the mpacks associated with the blueprint.
+ *
+ * @return associated mpacks
+ */
+ Collection<MpackInstance> getMpacks();
+
+ /**
* Get the host groups which contain components for the specified service.
*
* @param service service name
@@ -170,4 +196,15 @@ public interface Blueprint {
BlueprintEntity toEntity();
List<RepositorySetting> getRepositorySettings();
+
+ /**
+ * @return a boolean indicating if all mpack referenced by the blueprints are resolved (installed on the system)
+ */
+ boolean isAllMpacksResolved();
+
+ /**
+ * @return a collection of the names unresolved mpacks in {name}-{version} format.
+ */
+ Collection<String> getUnresolvedMpackNames();
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
index 24b4785..48f41a6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
@@ -19,8 +19,10 @@
package org.apache.ambari.server.topology;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -37,7 +39,12 @@ import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.orm.dao.BlueprintDAO;
import org.apache.ambari.server.orm.entities.BlueprintEntity;
import org.apache.ambari.server.stack.NoSuchStackException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Inject;
/**
@@ -45,6 +52,8 @@ import com.google.inject.Inject;
*/
public class BlueprintFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(BlueprintFactory.class);
+
// Blueprints
protected static final String BLUEPRINT_NAME_PROPERTY_ID =
PropertyHelper.getPropertyId("Blueprints", "blueprint_name");
@@ -70,6 +79,10 @@ public class BlueprintFactory {
protected static final String SETTINGS_PROPERTY_ID = "settings";
+ protected static final String MPACK_INSTANCES_PROPERTY = "mpack_instances";
+ protected static final String MPACK_INSTANCE_PROPERTY = "mpack_instance";
+ protected static final String SERVICE_INSTANCE_PROPERTY = "service_instance";
+
private static BlueprintDAO blueprintDAO;
private ConfigurationFactory configFactory = new ConfigurationFactory();
@@ -105,26 +118,77 @@ public class BlueprintFactory {
throw new IllegalArgumentException("Blueprint name must be provided");
}
- Stack stack = createStack(properties);
+ Stack stack;
+ Collection<MpackInstance> mpackInstances = createMpackInstances(properties);
+ if (mpackInstances.isEmpty()) {
+ stack = createStack(properties);
+ }
+ else {
+ stack = mpackInstances.iterator().next().getStack();
+ }
+
Collection<HostGroup> hostGroups = processHostGroups(name, stack, properties);
Configuration configuration = configFactory.getConfiguration((Collection<Map<String, String>>)
properties.get(CONFIGURATION_PROPERTY_ID));
- Setting setting = SettingFactory.getSetting((Collection<Map<String, Object>>) properties.get(SETTINGS_PROPERTY_ID));
+ Setting setting = SettingFactory.getSetting((Collection<Map<String, Object>>) properties.get(SETTINGS_PROPERTY_ID));
+
+ if (!mpackInstances.isEmpty()) {
+ return new BlueprintImpl(name, hostGroups, mpackInstances, configuration, securityConfiguration, setting);
+ }
+ else {
+ // Legacy constructor for old blueprints without mpacks
+ return new BlueprintImpl(name, hostGroups, stack, configuration, securityConfiguration, setting);
+ }
+ }
+
+ private Collection<MpackInstance> createMpackInstances(Map<String, Object> properties) throws NoSuchStackException {
+ if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+ try {
+ String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY));
+ Collection<MpackInstance> mpacks = mapper.readValue(mpackInstancesJson, new TypeReference<Collection<MpackInstance>>(){});
+ for (MpackInstance mpack: mpacks) {
+ tryResolveStack(mpack);
+ }
+ return mpacks;
+ }
+ catch (IOException ex) {
+ throw new RuntimeException("Unable to parse mpack instances for blueprint: " +
+ String.valueOf(properties.get(BLUEPRINT_NAME_PROPERTY_ID)), ex);
+ }
+ } else {
+ return Collections.emptyList();
+ }
+ }
- return new BlueprintImpl(name, hostGroups, stack, configuration, securityConfiguration, setting);
+ protected void tryResolveStack(MpackInstance mpack) {
+ try {
+ Stack stack = loadStack(mpack.getMpackName(), mpack.getMpackVersion());
+ mpack.setStack(stack);
+ }
+ catch (NoSuchStackException ex) {
+ // This case can be normal if a blueprint had been sent in before the referenced mpack was installed
+ LOG.warn("Cannot resolve stack for mpack {}-{}. Is mpack installed?", mpack.getMpackName(), mpack.getMpackVersion());
+ }
}
protected Stack createStack(Map<String, Object> properties) throws NoSuchStackException {
String stackName = String.valueOf(properties.get(STACK_NAME_PROPERTY_ID));
String stackVersion = String.valueOf(properties.get(STACK_VERSION_PROPERTY_ID));
+ return loadStack(stackName, stackVersion);
+ }
+
+ protected Stack loadStack(String stackName, String stackVersion) throws NoSuchStackException {
try {
//todo: don't pass in controller
return stackFactory.createStack(stackName, stackVersion, AmbariServer.getController());
} catch (ObjectNotFoundException e) {
throw new NoSuchStackException(stackName, stackVersion);
} catch (AmbariException e) {
- //todo:
- throw new RuntimeException("An error occurred parsing the stack information.", e);
+ // todo
+ throw new RuntimeException(
+ String.format("An error occurred parsing the stack information for %s-%s", stackName, stackVersion) , e);
}
}
@@ -155,7 +219,7 @@ public class BlueprintFactory {
Configuration configuration = configFactory.getConfiguration(configProps);
String cardinality = String.valueOf(hostGroupProperties.get(HOST_GROUP_CARDINALITY_PROPERTY_ID));
- HostGroup group = new HostGroupImpl(hostGroupName, bpName, stack, components, configuration, cardinality);
+ HostGroup group = new HostGroupImpl(hostGroupName, bpName, Collections.singleton(stack), components, configuration, cardinality);
hostGroups.add(group);
}
@@ -182,13 +246,12 @@ public class BlueprintFactory {
groupName + "' is not valid for the specified stack");
}
- String componentProvisionAction = componentProperties.get(COMPONENT_PROVISION_ACTION_PROPERTY_ID);
- if (componentProvisionAction != null) {
- //TODO, might want to add some validation here, to only accept value enum types, rwn
- components.add(new Component(componentName, ProvisionAction.valueOf(componentProvisionAction)));
- } else {
- components.add(new Component(componentName));
- }
+ String mpackInstance = componentProperties.get(MPACK_INSTANCE_PROPERTY);
+ String serviceInstance = componentProperties.get(SERVICE_INSTANCE_PROPERTY);
+ //TODO, might want to add some validation here, to only accept value enum types, rwn
+ ProvisionAction provisionAction = componentProperties.containsKey(COMPONENT_PROVISION_ACTION_PROPERTY_ID) ?
+ ProvisionAction.valueOf(componentProperties.get(COMPONENT_PROVISION_ACTION_PROPERTY_ID)) : null;
+ components.add(new Component(componentName, mpackInstance, serviceInstance, provisionAction));
}
return components;
@@ -219,7 +282,7 @@ public class BlueprintFactory {
*/
@Inject
public static void init(BlueprintDAO dao) {
- blueprintDAO = dao;
+ blueprintDAO = dao;
}
/**
@@ -229,7 +292,7 @@ public class BlueprintFactory {
* simulate various Stack or error conditions.
*/
interface StackFactory {
- Stack createStack(String stackName, String stackVersion, AmbariManagementController managementController) throws AmbariException;
+ Stack createStack(String stackName, String stackVersion, AmbariManagementController managementController) throws AmbariException;
}
/**
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
index 1552a6c..24e5d36 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
@@ -19,12 +19,17 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toList;
+
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.ambari.server.AmbariException;
@@ -34,15 +39,21 @@ import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.orm.entities.BlueprintConfigEntity;
import org.apache.ambari.server.orm.entities.BlueprintConfiguration;
import org.apache.ambari.server.orm.entities.BlueprintEntity;
+import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity;
+import org.apache.ambari.server.orm.entities.BlueprintServiceEntity;
import org.apache.ambari.server.orm.entities.BlueprintSettingEntity;
import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
import org.apache.ambari.server.orm.entities.HostGroupEntity;
-import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.stack.NoSuchStackException;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.base.Supplier;
import com.google.gson.Gson;
/**
@@ -50,51 +61,73 @@ import com.google.gson.Gson;
*/
public class BlueprintImpl implements Blueprint {
+ private static final Logger LOG = LoggerFactory.getLogger(BlueprintImpl.class);
+
private String name;
private Map<String, HostGroup> hostGroups = new HashMap<>();
- private Stack stack;
+ private Collection<MpackInstance> mpacks = new ArrayList<>();
private Configuration configuration;
private BlueprintValidator validator;
private SecurityConfiguration security;
private Setting setting;
- private List<RepositorySetting> repoSettings;
+ private List<RepositorySetting> repoSettings = new ArrayList<>();
+ private boolean allMpacksResolved = false;
public BlueprintImpl(BlueprintEntity entity) throws NoSuchStackException {
this.name = entity.getBlueprintName();
if (entity.getSecurityType() != null) {
- this.security = new SecurityConfiguration(entity.getSecurityType(), entity.getSecurityDescriptorReference(),
+ this.security = new SecurityConfiguration(entity.getSecurityType(),
+ entity.getSecurityDescriptorReference(),
null);
}
+ mpacks.addAll(parseMpacks(entity));
// create config first because it is set as a parent on all host-group configs
- processConfiguration(entity.getConfigurations());
+ configuration = processConfiguration(entity.getConfigurations());
parseBlueprintHostGroups(entity);
- configuration.setParentConfiguration(stack.getConfiguration(getServices()));
+ // TODO: how to handle multiple stacks correctly?
+// configuration.setParentConfiguration(stack.getConfiguration(getServices()));
validator = new BlueprintValidatorImpl(this);
processSetting(entity.getSettings());
processRepoSettings();
}
+ /**
+ * Legacy constructor for pre-multi-mpack code.
+ * @param name blueprint name
+ * @param groups host groups
+ * @param stack stack
+ * @param configuration configuration
+ * @param security security config
+ * @param setting setting
+ */
+ @Deprecated
public BlueprintImpl(String name, Collection<HostGroup> groups, Stack stack, Configuration configuration,
- SecurityConfiguration security) {
- this(name, groups, stack, configuration, security, null);
+ SecurityConfiguration security, Setting setting) {
+ this(name, groups, stackToMpacks(stack), configuration, security, setting);
}
- public BlueprintImpl(String name, Collection<HostGroup> groups, Stack stack, Configuration configuration,
- SecurityConfiguration security, Setting setting) {
+ private static Collection<MpackInstance> stackToMpacks(Stack stack) {
+ MpackInstance mpack = new MpackInstance(stack.getName(), stack.getVersion(), null, stack, new Configuration());
+ return Collections.singleton(mpack);
+ }
+
+ public BlueprintImpl(String name, Collection<HostGroup> groups, Collection<MpackInstance> mpacks,
+ Configuration configuration, SecurityConfiguration security, Setting setting) {
this.name = name;
- this.stack = stack;
+ this.mpacks = mpacks;
this.security = security;
// caller should set host group configs
for (HostGroup hostGroup : groups) {
hostGroups.put(hostGroup.getName(), hostGroup);
}
+ // TODO: handle configuration from multiple stacks properly
// if the parent isn't set, the stack configuration is set as the parent
this.configuration = configuration;
- if (configuration.getParentConfiguration() == null) {
- configuration.setParentConfiguration(stack.getConfiguration(getServices()));
- }
+// if (configuration.getParentConfiguration() == null) {
+// configuration.setParentConfiguration(stack.getConfiguration(getServices()));
+// }
validator = new BlueprintValidatorImpl(this);
this.setting = setting;
}
@@ -103,14 +136,6 @@ public class BlueprintImpl implements Blueprint {
return name;
}
- public String getStackName() {
- return stack.getName();
- }
-
- public String getStackVersion() {
- return stack.getVersion();
- }
-
public SecurityConfiguration getSecurity() {
return security;
}
@@ -152,15 +177,20 @@ public class BlueprintImpl implements Blueprint {
}
@Override
- public Collection<String> getComponents(String service) {
- Collection<String> components = new HashSet<>();
+ public Collection<Component> getComponents(String service) {
+ Collection<Component> components = new HashSet<>();
for (HostGroup group : getHostGroupsForService(service)) {
components.addAll(group.getComponents(service));
}
-
return components;
}
+ @Override
+ @Deprecated
+ public Collection<String> getComponentNames(String service) {
+ return getComponents(service).stream().map(Component::getName).collect(toList());
+ }
+
/**
* Get whether the specified component in the service is enabled
* for auto start.
@@ -273,14 +303,24 @@ public class BlueprintImpl implements Blueprint {
}
@Override
+ public Collection<MpackInstance> getMpacks() {
+ return mpacks;
+ }
+
+ @Override
+ public Collection<Stack> getStacks() {
+ return mpacks.stream().map(MpackInstance::getStack).filter(s -> null != s).collect(toList());
+ }
+
+ @Override
public Stack getStack() {
- return stack;
+ return getStacks().iterator().next();
}
/**
* Get host groups which contain a component.
*
- * @param component component name
+ * @param component component name
*
* @return collection of host groups which contain the specified component
*/
@@ -319,7 +359,6 @@ public class BlueprintImpl implements Blueprint {
}
public BlueprintEntity toEntity() {
-
BlueprintEntity entity = new BlueprintEntity();
entity.setBlueprintName(name);
if (security != null) {
@@ -332,12 +371,25 @@ public class BlueprintImpl implements Blueprint {
}
createHostGroupEntities(entity);
- createBlueprintConfigEntities(entity);
+ Collection<BlueprintConfigEntity> configEntities = toConfigEntities(getConfiguration(), () -> new BlueprintConfigEntity());
+ configEntities.forEach(configEntity -> {
+ configEntity.setBlueprintEntity(entity);
+ configEntity.setBlueprintName(getName());
+ });
+ entity.setConfigurations(configEntities);
createBlueprintSettingEntities(entity);
-
+ createMpackInstanceEntities(entity);
return entity;
}
+ private void createMpackInstanceEntities(BlueprintEntity entity) {
+ mpacks.forEach(mpack -> {
+ BlueprintMpackInstanceEntity mpackEntity = mpack.toEntity();
+ mpackEntity.setBlueprint(entity);
+ entity.getMpackInstances().add(mpackEntity);
+ });
+ }
+
/**
* Validate blueprint configuration.
*
@@ -349,21 +401,54 @@ public class BlueprintImpl implements Blueprint {
validator.validateRequiredProperties();
}
- private void parseStack(StackEntity stackEntity) throws NoSuchStackException {
+ private Collection<MpackInstance> parseMpacks(BlueprintEntity blueprintEntity) throws NoSuchStackException {
+ Collection<MpackInstance> mpackInstances = new ArrayList<>();
+ for (BlueprintMpackInstanceEntity mpack: blueprintEntity.getMpackInstances()) {
+ MpackInstance mpackInstance = new MpackInstance();
+ mpackInstance.setMpackName(mpack.getMpackName());
+ mpackInstance.setMpackVersion(mpack.getMpackVersion());
+ mpackInstance.setUrl(mpack.getMpackUri());
+ mpackInstance.setConfiguration(processConfiguration(mpack.getConfigurations()));
+ // TODO: come up with proper mpack -> stack resolution
+ tryParseStack(mpack.getMpackName(), mpack.getMpackVersion()).ifPresent( stack -> mpackInstance.setStack(stack) );
+ for(BlueprintServiceEntity serviceEntity: mpack.getServiceInstances()) {
+ ServiceInstance serviceInstance = new ServiceInstance(
+ serviceEntity.getName(),
+ serviceEntity.getType(),
+ processConfiguration(serviceEntity.getConfigurations()));
+ mpackInstance.addServiceInstance(serviceInstance);
+ }
+ mpackInstances.add(mpackInstance);
+ }
+ return mpackInstances;
+ }
+
+ private Stack parseStack(String stackName, String stackVersion) throws NoSuchStackException {
try {
//todo: don't pass in controller
- stack = new Stack(stackEntity.getStackName(), stackEntity.getStackVersion(), AmbariServer.getController());
+ return new Stack(stackName, stackVersion, AmbariServer.getController());
} catch (StackAccessException e) {
- throw new NoSuchStackException(stackEntity.getStackName(), stackEntity.getStackVersion());
+ throw new NoSuchStackException(stackName, stackVersion, e);
} catch (AmbariException e) {
//todo:
throw new RuntimeException("An error occurred parsing the stack information.", e);
}
}
+ private Optional<Stack> tryParseStack(String stackName, String stackVersion) {
+ try {
+ return Optional.of(parseStack(stackName, stackVersion));
+ }
+ catch (Exception ex) {
+ LOG.warn("Cannot parse stack {}-{}. Exception: {}/{}", stackName, stackVersion, ex.getClass().getName(),
+ ex.getMessage());
+ return Optional.empty();
+ }
+ }
+
private Map<String, HostGroup> parseBlueprintHostGroups(BlueprintEntity entity) {
for (HostGroupEntity hostGroupEntity : entity.getHostGroups()) {
- HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity, getName(), stack);
+ HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity, getName(), getStacks());
// set the bp configuration as the host group config parent
hostGroup.getConfiguration().setParentConfiguration(configuration);
hostGroups.put(hostGroupEntity.getName(), hostGroup);
@@ -374,9 +459,9 @@ public class BlueprintImpl implements Blueprint {
/**
* Process blueprint configurations.
*/
- private void processConfiguration(Collection<BlueprintConfigEntity> configs) {
+ private Configuration processConfiguration(Collection<? extends BlueprintConfiguration> configs) {
// not setting stack configuration as parent until after host groups are parsed in constructor
- configuration = new Configuration(parseConfigurations(configs),
+ return new Configuration(parseConfigurations(configs),
parseAttributes(configs), null);
}
@@ -396,8 +481,7 @@ public class BlueprintImpl implements Blueprint {
*
* @return map of config type to map of properties
*/
- private Map<String, Map<String, String>> parseConfigurations(Collection<BlueprintConfigEntity> configs) {
-
+ private Map<String, Map<String, String>> parseConfigurations(Collection<? extends BlueprintConfiguration> configs) {
Map<String, Map<String, String>> properties = new HashMap<>();
Gson gson = new Gson();
for (BlueprintConfiguration config : configs) {
@@ -414,7 +498,7 @@ public class BlueprintImpl implements Blueprint {
*
* @return map of setting name to map of properties
*/
- private Map<String, Set<HashMap<String, String>>> parseSetting(Collection<BlueprintSettingEntity> blueprintSetting) {
+ private Map<String, Set<HashMap<String, String>>> parseSetting(Collection<? extends BlueprintSettingEntity> blueprintSetting) {
Map<String, Set<HashMap<String, String>>> properties = new HashMap<>();
Gson gson = new Gson();
@@ -433,13 +517,13 @@ public class BlueprintImpl implements Blueprint {
* @return cluster scoped property attributes contained within in blueprint
*/
//todo: do inline with config processing
- private Map<String, Map<String, Map<String, String>>> parseAttributes(Collection<BlueprintConfigEntity> configs) {
+ private Map<String, Map<String, Map<String, String>>> parseAttributes(Collection<? extends BlueprintConfiguration> configs) {
Map<String, Map<String, Map<String, String>>> mapAttributes =
new HashMap<>();
if (configs != null) {
Gson gson = new Gson();
- for (BlueprintConfigEntity config : configs) {
+ for (BlueprintConfiguration config : configs) {
Map<String, Map<String, String>> typeAttrs =
gson.<Map<String, Map<String, String>>>fromJson(config.getConfigAttributes(), Map.class);
if (typeAttrs != null && !typeAttrs.isEmpty()) {
@@ -465,7 +549,13 @@ public class BlueprintImpl implements Blueprint {
hostGroupEntity.setBlueprintName(getName());
hostGroupEntity.setCardinality(group.getCardinality());
- createHostGroupConfigEntities(hostGroupEntity, group.getConfiguration());
+ Collection<HostGroupConfigEntity> configEntities = toConfigEntities(group.getConfiguration(), () -> new HostGroupConfigEntity());
+ configEntities.forEach(configEntity -> {
+ configEntity.setBlueprintName(getName());
+ configEntity.setHostGroupEntity(hostGroupEntity);
+ configEntity.setHostGroupName(group.getName());
+ });
+ hostGroupEntity.setConfigurations(configEntities);
createComponentEntities(hostGroupEntity, group.getComponents());
}
@@ -473,43 +563,6 @@ public class BlueprintImpl implements Blueprint {
}
/**
- * Populate host group configurations.
- */
- private void createHostGroupConfigEntities(HostGroupEntity hostGroup, Configuration groupConfiguration) {
- Gson jsonSerializer = new Gson();
- Map<String, HostGroupConfigEntity> configEntityMap = new HashMap<>();
- for (Map.Entry<String, Map<String, String>> propEntry : groupConfiguration.getProperties().entrySet()) {
- String type = propEntry.getKey();
- Map<String, String> properties = propEntry.getValue();
-
- HostGroupConfigEntity configEntity = new HostGroupConfigEntity();
- configEntityMap.put(type, configEntity);
- configEntity.setBlueprintName(getName());
- configEntity.setHostGroupEntity(hostGroup);
- configEntity.setHostGroupName(hostGroup.getName());
- configEntity.setType(type);
- configEntity.setConfigData(jsonSerializer.toJson(properties));
- }
-
- for (Map.Entry<String, Map<String, Map<String, String>>> attributesEntry : groupConfiguration.getAttributes().entrySet()) {
- String type = attributesEntry.getKey();
- Map<String, Map<String, String>> attributes = attributesEntry.getValue();
-
- HostGroupConfigEntity entity = configEntityMap.get(type);
- if (entity == null) {
- entity = new HostGroupConfigEntity();
- configEntityMap.put(type, entity);
- entity.setBlueprintName(getName());
- entity.setHostGroupEntity(hostGroup);
- entity.setHostGroupName(hostGroup.getName());
- entity.setType(type);
- }
- entity.setConfigAttributes(jsonSerializer.toJson(attributes));
- }
- hostGroup.setConfigurations(configEntityMap.values());
- }
-
- /**
* Create component entities and add to parent host group.
*/
@SuppressWarnings("unchecked")
@@ -525,6 +578,17 @@ public class BlueprintImpl implements Blueprint {
componentEntity.setBlueprintName(group.getBlueprintName());
componentEntity.setHostGroupEntity(group);
componentEntity.setHostGroupName(group.getName());
+ componentEntity.setServiceName(component.getServiceInstance());
+ if (null != component.getMpackInstance()) {
+ Preconditions.checkArgument(component.getMpackInstance().contains("-"),
+ "Invalid mpack instance specified for component %s: %s. Must be in {name}-{version} format.",
+ component.getName(),
+ component.getMpackInstance());
+ Iterator<String> mpackNameAndVersion =
+ Splitter.on('-').split(component.getMpackInstance()).iterator();
+ componentEntity.setMpackName(mpackNameAndVersion.next());
+ componentEntity.setMpackVersion(mpackNameAndVersion.next());
+ }
// add provision action (if specified) to entity type
// otherwise, just leave this column null (provision_action)
@@ -537,39 +601,62 @@ public class BlueprintImpl implements Blueprint {
}
/**
- * Populate host group configurations.
+ * Converts a {@link Configuration} class to a collection of configuration entities
+ * @param configuration the configuration to be converted
+ * @param entityCreator creates the appropriate config entity instance
+ * @param <E> the config entity's type
+ * @return a collection of configuration entities
*/
- private void createBlueprintConfigEntities(BlueprintEntity blueprintEntity) {
+ static <E extends BlueprintConfiguration> Collection<E> toConfigEntities(Configuration configuration, Supplier<E> entityCreator) {
Gson jsonSerializer = new Gson();
- Configuration config = getConfiguration();
- Map<String, BlueprintConfigEntity> configEntityMap = new HashMap<>();
- for (Map.Entry<String, Map<String, String>> propEntry : config.getProperties().entrySet()) {
+ Map<String, E> configEntityMap = new HashMap<>();
+ for (Map.Entry<String, Map<String, String>> propEntry : configuration.getProperties().entrySet()) {
String type = propEntry.getKey();
Map<String, String> properties = propEntry.getValue();
- BlueprintConfigEntity configEntity = new BlueprintConfigEntity();
+ E configEntity = entityCreator.get();
configEntityMap.put(type, configEntity);
- configEntity.setBlueprintName(getName());
- configEntity.setBlueprintEntity(blueprintEntity);
configEntity.setType(type);
configEntity.setConfigData(jsonSerializer.toJson(properties));
}
- for (Map.Entry<String, Map<String, Map<String, String>>> attributesEntry : config.getAttributes().entrySet()) {
+ for (Map.Entry<String, Map<String, Map<String, String>>> attributesEntry : configuration.getAttributes().entrySet()) {
String type = attributesEntry.getKey();
Map<String, Map<String, String>> attributes = attributesEntry.getValue();
- BlueprintConfigEntity entity = configEntityMap.get(type);
+ E entity = configEntityMap.get(type);
if (entity == null) {
- entity = new BlueprintConfigEntity();
+ entity = entityCreator.get();
configEntityMap.put(type, entity);
- entity.setBlueprintName(getName());
- entity.setBlueprintEntity(blueprintEntity);
entity.setType(type);
}
entity.setConfigAttributes(jsonSerializer.toJson(attributes));
}
- blueprintEntity.setConfigurations(configEntityMap.values());
+ return configEntityMap.values();
+ }
+
+ /**
+ * Converts a collection of configuration entities into a {@link Configuration}
+ * @param configEntities the persisted configuration as entitiy collection
+ * @return the configuration
+ */
+ static Configuration fromConfigEntities(Collection<? extends BlueprintConfiguration> configEntities) {
+ Gson jsonSerializer = new Gson();
+ Configuration configuration = new Configuration();
+
+ for (BlueprintConfiguration configEntity: configEntities) {
+ String type = configEntity.getType();
+ Map<String, String> configData = jsonSerializer.fromJson(configEntity.getConfigData(), Map.class);
+ Map<String, Map<String, String>> configAttributes = jsonSerializer.fromJson(configEntity.getConfigAttributes(), Map.class);
+ if (null != configData) {
+ configuration.getProperties().put(type, configData);
+ }
+ if (null != configAttributes) {
+ configuration.getAttributes().put(type, configAttributes);
+ }
+ }
+
+ return configuration;
}
/**
@@ -602,9 +689,12 @@ public class BlueprintImpl implements Blueprint {
if (ConfigHelper.CLUSTER_ENV.equals(configType) || "global".equals(configType)) {
return true;
}
- String service = getStack().getServiceForConfigType(configType);
- if (getServices().contains(service)) {
+
+ Collection<String> services = getStacks().stream().map(stack -> stack.getServiceForConfigType(configType)).collect(toList());
+ for (String service: services) {
+ if (getServices().contains(service)) {
return true;
+ }
}
return false;
}
@@ -636,4 +726,20 @@ public class BlueprintImpl implements Blueprint {
public List<RepositorySetting> getRepositorySettings(){
return repoSettings;
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isAllMpacksResolved() {
+ return !mpacks.stream().filter(mpack -> mpack.getStack() == null).findAny().isPresent();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public Collection<String> getUnresolvedMpackNames() {
+ return mpacks.stream().filter(mpack -> mpack.getStack() == null).map(MpackInstance::getMpackNameAndVersion).collect(toList());
+ }
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index fbd0e4b..1e6213e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -36,6 +36,7 @@ import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Joiner;
import com.google.inject.Inject;
/**
@@ -63,6 +64,13 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
@Override
public void validateTopology() throws InvalidTopologyException {
LOGGER.info("Validating topology for blueprint: [{}]", blueprint.getName());
+
+ if (!blueprint.isAllMpacksResolved()) {
+ LOGGER.warn("The following macks are not resolved: [{}] Skipping topology validation.",
+ Joiner.on(", ").join(blueprint.getUnresolvedMpackNames()));
+ return;
+ }
+
Collection<HostGroup> hostGroups = blueprint.getHostGroups().values();
Map<String, Map<String, Collection<DependencyInfo>>> missingDependencies = new HashMap<>();
@@ -81,10 +89,10 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
Cardinality cardinality = stack.getCardinality(component);
AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component);
if (cardinality.isAll()) {
- cardinalityFailures.addAll(verifyComponentInAllHostGroups(component, autoDeploy));
+ cardinalityFailures.addAll(verifyComponentInAllHostGroups(new Component(component), autoDeploy));
} else {
cardinalityFailures.addAll(verifyComponentCardinalityCount(
- component, cardinality, autoDeploy));
+ new Component(component), cardinality, autoDeploy));
}
}
}
@@ -178,27 +186,33 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
}
}
- if (component.equals("HIVE_METASTORE")) {
- Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env");
- if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null
- && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database")
- && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
- && stack.getName().equalsIgnoreCase("HDP")) {
- throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " +
- "and repo version 2.3.2+!");
+ if (blueprint.isAllMpacksResolved()) {
+ if (component.equals("HIVE_METASTORE")) {
+ Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env");
+ if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null
+ && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database")
+ && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
+ && stack.getName().equalsIgnoreCase("HDP")) {
+ throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " +
+ "and repo version 2.3.2+!");
+ }
}
- }
- if (component.equals("OOZIE_SERVER")) {
- Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env");
- if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") != null
- && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database")
- && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
- && stack.getName().equalsIgnoreCase("HDP")) {
- throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " +
- "and repo version 2.3.2+!");
+ if (component.equals("OOZIE_SERVER")) {
+ Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env");
+ if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") != null
+ && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database")
+ && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
+ && stack.getName().equalsIgnoreCase("HDP")) {
+ throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " +
+ "and repo version 2.3.2+!");
+ }
}
}
+ else {
+ LOGGER.warn("The following macks are not resolved: [{}] Skipping validation of HIVE_METASTORE and OOZIE_SERVER properties.",
+ Joiner.on(", ").join(blueprint.getUnresolvedMpackNames()));
+ }
}
}
}
@@ -212,10 +226,10 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
*
* @return collection of missing component information
*/
- private Collection<String> verifyComponentInAllHostGroups(String component, AutoDeployInfo autoDeploy) {
+ private Collection<String> verifyComponentInAllHostGroups(Component component, AutoDeployInfo autoDeploy) {
Collection<String> cardinalityFailures = new HashSet<>();
- int actualCount = blueprint.getHostGroupsForComponent(component).size();
+ int actualCount = blueprint.getHostGroupsForComponent(component.getName()).size();
Map<String, HostGroup> hostGroups = blueprint.getHostGroups();
if (actualCount != hostGroups.size()) {
if (autoDeploy != null && autoDeploy.isEnabled()) {
@@ -233,10 +247,10 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
LOGGER.info("Validating hostgroup: {}", group.getName());
Map<String, Collection<DependencyInfo>> missingDependencies = new HashMap<>();
- for (String component : new HashSet<>(group.getComponentNames())) {
+ for (Component component : new HashSet<>(group.getComponents())) {
LOGGER.debug("Processing component: {}", component);
- for (DependencyInfo dependency : stack.getDependenciesForComponent(component)) {
+ for (DependencyInfo dependency : stack.getDependenciesForComponent(component.getName())) {
LOGGER.debug("Processing dependency [{}] for component [{}]", dependency.getName(), component);
String conditionalService = stack.getConditionalServiceForDependency(dependency);
@@ -274,21 +288,21 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
}
if (dependencyScope.equals("cluster")) {
Collection<String> missingDependencyInfo = verifyComponentCardinalityCount(
- componentName, new Cardinality("1+"), autoDeployInfo);
+ new Component(componentName), new Cardinality("1+"), autoDeployInfo);
resolved = missingDependencyInfo.isEmpty();
} else if (dependencyScope.equals("host")) {
if (group.getComponentNames().contains(componentName) || (autoDeployInfo != null && autoDeployInfo.isEnabled())) {
resolved = true;
- group.addComponent(componentName);
+ group.addComponent(new Component(componentName));
}
}
if (! resolved) {
- Collection<DependencyInfo> missingCompDependencies = missingDependencies.get(component);
+ Collection<DependencyInfo> missingCompDependencies = missingDependencies.get(component.getName());
if (missingCompDependencies == null) {
missingCompDependencies = new HashSet<>();
- missingDependencies.put(component, missingCompDependencies);
+ missingDependencies.put(component.getName(), missingCompDependencies);
}
missingCompDependencies.add(dependency);
}
@@ -307,7 +321,7 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
*
* @return collection of missing component information
*/
- public Collection<String> verifyComponentCardinalityCount(String component,
+ public Collection<String> verifyComponentCardinalityCount(Component component,
Cardinality cardinality,
AutoDeployInfo autoDeploy) {
@@ -315,15 +329,15 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
Collection<String> cardinalityFailures = new HashSet<>();
//todo: don't hard code this HA logic here
if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) &&
- (component.equals("SECONDARY_NAMENODE"))) {
+ (component.getName().equals("SECONDARY_NAMENODE"))) {
// override the cardinality for this component in an HA deployment,
// since the SECONDARY_NAMENODE should not be started in this scenario
cardinality = new Cardinality("0");
}
- int actualCount = blueprint.getHostGroupsForComponent(component).size();
+ int actualCount = blueprint.getHostGroupsForComponent(component.getName()).size();
if (! cardinality.isValidCount(actualCount)) {
- boolean validated = ! isDependencyManaged(stack, component, configProperties);
+ boolean validated = ! isDependencyManaged(stack, component.getName(), configProperties);
if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) {
String coLocateName = autoDeploy.getCoLocate();
if (coLocateName != null && ! coLocateName.isEmpty()) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 740dd91..dfe5dd0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -237,11 +237,11 @@ public class ClusterConfigurationRequest {
if(services != null) {
for (String service : services) {
- Collection<String> components = blueprint.getComponents(service);
+ Collection<String> components = blueprint.getComponentNames(service);
serviceComponents.put(service,
(components == null)
? Collections.emptySet()
- : new HashSet<>(blueprint.getComponents(service)));
+ : new HashSet<>(blueprint.getComponentNames(service)));
}
}
@@ -281,7 +281,7 @@ public class ClusterConfigurationRequest {
private Map<String, String> createComponentHostMap(Blueprint blueprint) {
Map<String, String> componentHostsMap = new HashMap<>();
for (String service : blueprint.getServices()) {
- Collection<String> components = blueprint.getComponents(service);
+ Collection<String> components = blueprint.getComponentNames(service);
for (String component : components) {
Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component);
// retrieve corresponding clusterInfoKey for component using StageUtils
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java
index 03620ad..a62d341 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java
@@ -19,21 +19,31 @@
package org.apache.ambari.server.topology;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.controller.internal.ProvisionAction;
public class Component {
private final String name;
+ @Nullable
+ private final String mpackInstance;
+ @Nullable
+ private final String serviceInstance;
private final ProvisionAction provisionAction;
+ @Deprecated
public Component(String name) {
- this(name, null);
+ this(name, null, null, null);
}
-
- public Component(String name, ProvisionAction provisionAction) {
+ public Component(String name, String mpackInstance, String serviceInstance, ProvisionAction provisionAction) {
this.name = name;
+ this.mpackInstance = mpackInstance;
+ this.serviceInstance = serviceInstance;
this.provisionAction = provisionAction;
}
@@ -47,6 +57,21 @@ public class Component {
}
/**
+ * @return the mpack associated with this component (can be {@code null} if component -> mpack mapping is unambigous)
+ */
+ public String getMpackInstance() {
+ return mpackInstance;
+ }
+
+ /**
+ * @return the service instance this component belongs to. Can be {@null} if component does not belong to a service
+ * instance (there is a single service of the component's service type)
+ */
+ public String getServiceInstance() {
+ return serviceInstance;
+ }
+
+ /**
* Gets the provision action associated with this component.
*
* @return the provision action for this component, which
@@ -56,4 +81,30 @@ public class Component {
return this.provisionAction;
}
+ @Override
+ public String toString() {
+ return com.google.common.base.Objects.toStringHelper(this)
+ .add("name", name)
+ .add("mpackInstance", mpackInstance)
+ .add("serviceInstance", serviceInstance)
+ .add("provisionAction", provisionAction)
+ .toString();
+ }
+
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Component component = (Component) o;
+ return Objects.equals(name, component.name) &&
+ Objects.equals(mpackInstance, component.mpackInstance) &&
+ Objects.equals(serviceInstance, component.serviceInstance) &&
+ provisionAction == component.provisionAction;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, mpackInstance, serviceInstance, provisionAction);
+ }
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configurable.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configurable.java
new file mode 100644
index 0000000..b71b12c
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configurable.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import static org.apache.ambari.server.topology.BlueprintFactory.PROPERTIES_ATTRIBUTES_PROPERTY_ID;
+import static org.apache.ambari.server.topology.BlueprintFactory.PROPERTIES_PROPERTY_ID;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+
+
+
+public interface Configurable {
+
+ @JsonIgnore
+ void setConfiguration(Configuration configuration);
+ @JsonIgnore
+ Configuration getConfiguration();
+
+ @JsonProperty("configurations")
+ default void setConfigs(Collection<? extends Map<String, ? extends Object>> configs) {
+ Configuration configuration;
+ if (!configs.isEmpty() && configs.iterator().next().keySet().iterator().next().contains("/")) {
+ // Configuration has keys with slashes like "zk.cfg/properties/dataDir" means it is coming through
+ // the resource framework and must be parsed with configuration factories
+ configuration = new ConfigurationFactory().getConfiguration((Collection<Map<String, String>>)configs);
+ }
+ else {
+ // If the configuration does not have keys with slashes it means it is coming from plain JSON and needs to be
+ // parsed accordingly.
+ Map<String, Map<String, String>> allProperties = new HashMap<>();
+ Map<String, Map<String, Map<String, String>>> allAttributes = new HashMap<>();
+ configs.forEach( item -> {
+ String configName = item.keySet().iterator().next();
+ Map<String, Object> configData = (Map<String, Object>) item.get(configName);
+ if (configData.containsKey(PROPERTIES_PROPERTY_ID)) {
+ Map<String, String> properties = (Map<String, String>)configData.get(PROPERTIES_PROPERTY_ID);
+ allProperties.put(configName, properties);
+ }
+ if (configData.containsKey(PROPERTIES_ATTRIBUTES_PROPERTY_ID)) {
+ Map<String, Map<String, String>> attributes =
+ (Map<String, Map<String, String>>)configData.get(PROPERTIES_ATTRIBUTES_PROPERTY_ID);
+ allAttributes.put(configName, attributes);
+ }
+ });
+ configuration = new Configuration(allProperties, allAttributes);
+ }
+ setConfiguration(configuration);
+ }
+
+ @JsonProperty("configurations")
+ default Collection<Map<String, Map<String, ? extends Object>>> getConfigs() {
+ Configuration configuration = getConfiguration();
+ Collection<Map<String, Map<String, ? extends Object>>> configurations = new ArrayList<>();
+ Set<String> allConfigTypes = Sets.union(configuration.getProperties().keySet(), configuration.getAttributes().keySet());
+ for (String configType: allConfigTypes) {
+ Map<String, Map<String, ? extends Object>> configData = new HashMap<>();
+ if (configuration.getProperties().containsKey(configType)) {
+ configData.put(PROPERTIES_PROPERTY_ID, configuration.getProperties().get(configType));
+ }
+ if (configuration.getAttributes().containsKey(configType)) {
+ configData.put(PROPERTIES_ATTRIBUTES_PROPERTY_ID, configuration.getAttributes().get(configType));
+ }
+ configurations.add(ImmutableMap.of(configType, configData));
+ }
+ return configurations;
+ }
+
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 28b62bc..8ffbdae 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -43,6 +43,12 @@ public class Configuration {
*/
private Configuration parentConfiguration;
+
+ public Configuration() {
+ properties = new HashMap<>();
+ attributes = new HashMap<>();
+ }
+
/**
* Constructor.
*
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java
index 3e80247..43ce5b9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java
@@ -62,11 +62,13 @@ public interface HostGroup {
*/
Collection<Component> getComponents();
+
/**
* Get all of the host group component names
*
* @return collection of component names as String
*/
+ @Deprecated
Collection<String> getComponentNames();
/**
@@ -79,35 +81,38 @@ public interface HostGroup {
* @return collection of component names as String that are associated with
* the specified provision action
*/
+ @Deprecated
Collection<String> getComponentNames(ProvisionAction provisionAction);
/**
- * Get the host group components which belong to the specified service.
+ * Get the names components for the specified service which are associated with the host group.
*
* @param service service name
*
- * @return collection of component names for the specified service; will not return null
+ * @return set of component names
*/
- Collection<String> getComponents(String service);
+ @Deprecated
+ public Collection<String> getComponentNames(String service);
/**
- * Add a component to the host group.
+ * Get the host group components which belong to the specified service.
*
- * @param component name of the component to add
+ * @param service service instance name or service name. First, services looked up
+ * by instance name. If no appropriate service instance is found, services are looked
+ * up by type
*
- * @return true if the component didn't already exist
+ * @return collection of component names for the specified service; will not return null
*/
- boolean addComponent(String component);
+ Collection<Component> getComponents(String service);
+
+
/**
- * Add a component to the host group, with the specified name
- * and provision action.
- *
- * @param component component name
- * @param provisionAction provision action for this component
+ * Add a component to the host group
+ * @param component
* @return
*/
- boolean addComponent(String component, ProvisionAction provisionAction);
+ boolean addComponent(Component component);
/**
* Determine if the host group contains a master component.
@@ -137,9 +142,17 @@ public interface HostGroup {
*
* @return associated stack
*/
+ @Deprecated
Stack getStack();
/**
+ * Get the stack associated with the host group.
+ *
+ * @return associated stacks
+ */
+ Collection<Stack> getStacks();
+
+ /**
* Get the cardinality value that was specified for the host group.
* This is simply meta-data for the stack that a deployer can use
* and this information is not used by ambari.
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java
index 9aeadd1..88ad162 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java
@@ -19,12 +19,19 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.ambari.server.controller.internal.ProvisionAction;
import org.apache.ambari.server.controller.internal.Stack;
@@ -32,6 +39,10 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
import org.apache.ambari.server.orm.entities.HostGroupEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
import com.google.gson.Gson;
/**
@@ -39,6 +50,8 @@ import com.google.gson.Gson;
*/
public class HostGroupImpl implements HostGroup {
+ private static final Logger LOG = LoggerFactory.getLogger(HostGroupImpl.class);
+
/**
* host group name
*/
@@ -52,12 +65,13 @@ public class HostGroupImpl implements HostGroup {
/**
* components contained in the host group
*/
- private Map<String, Component> components = new HashMap<>();
+ private final List<Component> components = new ArrayList<>();
/**
* map of service to components for the host group
*/
- private Map<String, Set<String>> componentsForService = new HashMap<>();
+ // TODO: in blueprint 3.0 this should be per service instance
+ private Map<String, Set<Component>> componentsForService = new HashMap<>();
/**
* configuration
@@ -66,28 +80,29 @@ public class HostGroupImpl implements HostGroup {
private boolean containsMasterComponent = false;
- private Stack stack;
+ private Map<String, Stack> stackMap;
private String cardinality = "NOT SPECIFIED";
- public HostGroupImpl(HostGroupEntity entity, String blueprintName, Stack stack) {
+ public HostGroupImpl(HostGroupEntity entity, String blueprintName, Collection<Stack> stacks) {
this.name = entity.getName();
this.cardinality = entity.getCardinality();
this.blueprintName = blueprintName;
- this.stack = stack;
-
+ this.stackMap = stacks.stream().collect(Collectors.toMap(stack -> stack.getName() + "-" + stack.getVersion(), stack -> stack));
parseComponents(entity);
parseConfigurations(entity);
}
- public HostGroupImpl(String name, String bpName, Stack stack, Collection<Component> components, Configuration configuration, String cardinality) {
+ public HostGroupImpl(String name, String bpName, Collection<Stack> stacks, Collection<Component> components, Configuration configuration, String cardinality) {
this.name = name;
this.blueprintName = bpName;
- this.stack = stack;
+ this.stackMap = stacks.stream().collect(Collectors.toMap(stack -> stack.getName() + "-" + stack.getVersion(), stack -> stack));
// process each component
- for (Component component : components) {
- addComponent(component.getName(), component.getProvisionAction());
+ for (Component component: components) {
+ if (!addComponent(component)) {
+ throw new IllegalArgumentException("Ambiguous component or can't determine stack for: " + component);
+ }
}
this.configuration = configuration;
@@ -115,28 +130,27 @@ public class HostGroupImpl implements HostGroup {
@Override
public Collection<Component> getComponents() {
- return components.values();
+ return components;
}
@Override
+ @Deprecated
public Collection<String> getComponentNames() {
- return components.keySet();
+ return components.stream().map(Component::getName).collect(toList());
}
@Override
+ @Deprecated
public Collection<String> getComponentNames(ProvisionAction provisionAction) {
Set<String> setOfComponentNames = new HashSet<>();
- for (String componentName : components.keySet()) {
- Component component = components.get(componentName);
+ for (Component component : components) {
if ( (component.getProvisionAction() != null) && (component.getProvisionAction() == provisionAction) ) {
- setOfComponentNames.add(componentName);
+ setOfComponentNames.add(component.getName());
}
}
-
return setOfComponentNames;
}
-
/**
* Get the services which are deployed to this host group.
*
@@ -148,51 +162,82 @@ public class HostGroupImpl implements HostGroup {
}
/**
- * Add a component to the host group.
- *
- * @param component component to add
- *
- * @return true if component was added; false if component already existed
+ * Adds a component to the host group. The component is successfully added if it is not a duplicate or ambiguous (as of
+ * Ambari 3.1 multiple components of the same type can exist in a hostgroup. However, they have to come from different
+ * management packs or belong to different service instances)
+ * @param component the component to add
+ * @return a boolean to indicate if addition was successful
*/
- @Override
- public boolean addComponent(String component) {
- return this.addComponent(component, null);
+ public boolean addComponent(Component component) {
+ // Exclude ambiguous component definitions
+ boolean ambigous = components.stream().filter(c -> {
+ if (c.getName().equals(component.getName())) { // found another component with the same name
+ if (c.getMpackInstance() == null || component.getMpackInstance() == null) {
+ return true; // if either of them has no mpack instance defined it is ambiguous
+ }
+ if (c.getMpackInstance().equals(component.getMpackInstance())) {
+ // both components are in the same mpack, and one of them does not declare a service instance or
+ // both declare the same service instance --> ambiguous
+ return c.getServiceInstance() != null && component.getServiceInstance() != null &&
+ c.getServiceInstance().equals(component.getServiceInstance());
+ }
+ else {
+ return false; // different mpacks --> no ambiguity
+ }
+ }
+ else {
+ return false; // different name --> no ambiguity
+ }
+ }).findAny().isPresent();
+ if (ambigous) {
+ return false;
+ }
+ addComponent(component, getStackForComponent(component));
+ return true;
}
- /**
- * Add a component with the specified provision action to the
- * host group.
- *
- * @param component component name
- * @param provisionAction provision action for this component
- *
- * @return true if component was added; false if component already existed
- */
- public boolean addComponent(String component, ProvisionAction provisionAction) {
- boolean added;
- if (!components.containsKey(component)) {
- components.put(component, new Component(component, provisionAction));
- added = true;
- } else {
- added = false;
+ private Optional<Stack> getStackForComponent(Component component) {
+ // Look for the stack of this component
+ if (component.getMpackInstance() == null) {
+ // Component does not declare its stack. Let's find it.
+ Collection<Stack> candidateStacks =
+ stackMap.values().stream().filter(stack -> stack.getServiceForComponent(component.getName()) != null).collect(toList());
+ switch (candidateStacks.size()) {
+ case 0:
+ // no stack (no service) for this component
+ LOG.warn("No stack/service found for component: {}", component);
+ return Optional.empty();
+ case 1:
+ return Optional.of(candidateStacks.iterator().next());
+ default:
+ LOG.warn("Ambiguous stack resolution for component: {}, stacks: {}", component, candidateStacks);
+ return Optional.empty();
+ }
}
-
- if (stack.isMasterComponent(component)) {
- containsMasterComponent = true;
+ else {
+ // TODO: refine this logic
+ Stack stack = stackMap.get(component.getMpackInstance());
+ if (null == stack) {
+ LOG.warn("Component declared an invalid stack: {}", component);
+ }
+ return Optional.ofNullable(stack);
}
- if (added) {
- String service = stack.getServiceForComponent(component);
- if (service != null) {
- // an example of a component without a service in the stack is AMBARI_SERVER
- Set<String> serviceComponents = componentsForService.get(service);
- if (serviceComponents == null) {
- serviceComponents = new HashSet<>();
- componentsForService.put(service, serviceComponents);
- }
- serviceComponents.add(component);
+ }
+
+ private void addComponent(Component component, Optional<Stack> stack) {
+ if (stack.isPresent()) {
+ String serviceName = stack.get().getServiceForComponent(component.getName());
+ if (!componentsForService.containsKey(serviceName)) {
+ componentsForService.put(serviceName, Sets.newHashSet(component));
+ }
+ else {
+ componentsForService.get(serviceName).add(component);
+ }
+ if (stack.get().isMasterComponent(component.getName())) {
+ containsMasterComponent = true;
}
}
- return added;
+ components.add(component);
}
/**
@@ -200,23 +245,38 @@ public class HostGroupImpl implements HostGroup {
*
* @param service service name
*
- * @return set of component names
+ * @return set of components
*/
@Override
- public Collection<String> getComponents(String service) {
+ public Collection<Component> getComponents(String service) {
return componentsForService.containsKey(service) ?
new HashSet<>(componentsForService.get(service)) :
Collections.emptySet();
}
/**
+ * Get the names components for the specified service which are associated with the host group.
+ *
+ * @param service service name
+ *
+ * @return set of component names
+ */
+ @Override
+ @Deprecated
+ public Collection<String> getComponentNames(String service) {
+ return componentsForService.containsKey(service) ?
+ componentsForService.get(service).stream().map(Component::getName).collect(toSet()) :
+ Collections.emptySet();
+ }
+
+
+ /**
* Get this host groups configuration.
*
* @return configuration instance
*/
@Override
public Configuration getConfiguration() {
-
return configuration;
}
@@ -236,8 +296,14 @@ public class HostGroupImpl implements HostGroup {
}
@Override
+ public Collection<Stack> getStacks() {
+ return stackMap.values();
+ }
+
+ @Override
+ @Deprecated
public Stack getStack() {
- return stack;
+ return getStacks().iterator().next();
}
@Override
@@ -250,13 +316,12 @@ public class HostGroupImpl implements HostGroup {
*/
private void parseComponents(HostGroupEntity entity) {
for (HostGroupComponentEntity componentEntity : entity.getComponents() ) {
- if (componentEntity.getProvisionAction() != null) {
- addComponent(componentEntity.getName(), ProvisionAction.valueOf(componentEntity.getProvisionAction()));
- } else {
- addComponent(componentEntity.getName());
- }
-
-
+ Component component = new Component(
+ componentEntity.getName(),
+ componentEntity.getMpackName(),
+ componentEntity.getServiceName(),
+ null == componentEntity.getProvisionAction() ? null : ProvisionAction.valueOf(componentEntity.getProvisionAction()));
+ addComponent(component);
}
}
@@ -270,13 +335,12 @@ public class HostGroupImpl implements HostGroup {
for (HostGroupConfigEntity configEntity : entity.getConfigurations()) {
String type = configEntity.getType();
Map<String, String> typeProperties = config.get(type);
- if ( typeProperties == null) {
+ if (typeProperties == null) {
typeProperties = new HashMap<>();
config.put(type, typeProperties);
}
Map<String, String> propertyMap = jsonSerializer.<Map<String, String>>fromJson(
configEntity.getConfigData(), Map.class);
-
if (propertyMap != null) {
typeProperties.putAll(propertyMap);
}
@@ -285,6 +349,7 @@ public class HostGroupImpl implements HostGroup {
Map<String, Map<String, Map<String, String>>> attributes = new HashMap<>();
configuration = new Configuration(config, attributes);
}
+
public String toString(){
return name;
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
index 4648412..b00faad 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -76,6 +77,8 @@ public class HostGroupInfo {
*/
Predicate predicate;
+ private Collection<ManagementPackMapping> managementPackMappings = new ArrayList<>();
+
/**
* Constructor
@@ -242,4 +245,11 @@ public class HostGroupInfo {
}
}
+ public Collection<ManagementPackMapping> getManagementPackMappings() {
+ return managementPackMappings;
+ }
+
+ public void setManagementPackMappings(Collection<ManagementPackMapping> managementPackMappings) {
+ this.managementPackMappings = managementPackMappings;
+ }
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ManagementPackMapping.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ManagementPackMapping.java
new file mode 100644
index 0000000..334991b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ManagementPackMapping.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Iterator;
+
+import com.google.common.base.Splitter;
+
+/**
+ * Represents a component -> mpack mapping in the multi-everything cluster topology template.
+ */
+public class ManagementPackMapping {
+ private String componentName;
+ private String mpackName;
+ private String mpackVersion;
+
+ public ManagementPackMapping(String componentName, String mpackNameAndVersion) {
+ this.componentName = componentName;
+ checkArgument(mpackNameAndVersion.contains("-"),
+ "Management pack must be specified in {name}-{version} format. Received %s", mpackNameAndVersion);
+ Iterator<String> mpackNameIterator = Splitter.on('-').split(mpackNameAndVersion).iterator();
+ this.mpackName = mpackNameIterator.next();
+ this.mpackVersion = mpackNameIterator.next();
+ }
+
+ public String getComponentName() {
+ return componentName;
+ }
+
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ public String getMpackName() {
+ return mpackName;
+ }
+
+ public void setMpackName(String mpackName) {
+ this.mpackName = mpackName;
+ }
+
+ public String getMpackVersion() {
+ return mpackVersion;
+ }
+
+ public void setMpackVersion(String mpackVersion) {
+ this.mpackVersion = mpackVersion;
+ }
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java
new file mode 100644
index 0000000..cde01e0
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.orm.entities.BlueprintMpackConfigEntity;
+import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity;
+import org.apache.ambari.server.orm.entities.BlueprintServiceConfigEntity;
+import org.apache.ambari.server.orm.entities.BlueprintServiceEntity;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class MpackInstance implements Configurable {
+ @JsonProperty("name")
+ private String mpackName;
+ @JsonProperty("version")
+ private String mpackVersion;
+ @JsonProperty("url")
+ private String url;
+
+ private Stack stack;
+ private Configuration configuration = new Configuration();
+
+ @JsonProperty("service_instances")
+ private Collection<ServiceInstance> serviceInstances = new ArrayList<>();
+
+ public MpackInstance(String mpackName, String mpackVersion, String url, Stack stack, Configuration configuration) {
+ this.mpackName = mpackName;
+ this.mpackVersion = mpackVersion;
+ this.url = url;
+ this.stack = stack;
+ this.configuration = configuration;
+ }
+
+ public MpackInstance() { }
+
+ public String getMpackName() {
+ return mpackName;
+ }
+
+ public void setMpackName(String mpackName) {
+ this.mpackName = mpackName;
+ }
+
+ public String getMpackVersion() {
+ return mpackVersion;
+ }
+
+ public void setMpackVersion(String mpackVersion) {
+ this.mpackVersion = mpackVersion;
+ }
+
+ @JsonIgnore
+ public String getMpackNameAndVersion() {
+ return mpackName + "-" + mpackVersion;
+ }
+
+ @JsonIgnore
+ public Stack getStack() {
+ return stack;
+ }
+
+ public void setStack(Stack stack) {
+ this.stack = stack;
+ }
+
+ @JsonIgnore
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @JsonIgnore
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ public Collection<ServiceInstance> getServiceInstances() {
+ return serviceInstances;
+ }
+
+ public void setServiceInstances(Collection<ServiceInstance> serviceInstances) {
+ this.serviceInstances = serviceInstances;
+ serviceInstances.forEach(si -> si.setMpackInstance(this));
+ }
+
+ public void addServiceInstance(ServiceInstance serviceInstance) {
+ serviceInstances.add(serviceInstance);
+ serviceInstance.setMpackInstance(this);
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public BlueprintMpackInstanceEntity toEntity() {
+ BlueprintMpackInstanceEntity mpackEntity = new BlueprintMpackInstanceEntity();
+ mpackEntity.setMpackUri(url);
+ mpackEntity.setMpackName(mpackName);
+ mpackEntity.setMpackVersion(mpackVersion);
+ Collection<BlueprintMpackConfigEntity> mpackConfigEntities =
+ BlueprintImpl.toConfigEntities(configuration, () -> new BlueprintMpackConfigEntity());
+ mpackConfigEntities.forEach( configEntity -> configEntity.setMpackInstance(mpackEntity) );
+ mpackEntity.setConfigurations(mpackConfigEntities);
+
+ getServiceInstances().forEach(serviceInstance -> {
+ BlueprintServiceEntity serviceEntity = new BlueprintServiceEntity();
+ serviceEntity.setName(serviceInstance.getName());
+ serviceEntity.setType(serviceInstance.getType());
+ Collection<BlueprintServiceConfigEntity> serviceConfigEntities =
+ BlueprintImpl.toConfigEntities(serviceInstance.getConfiguration(), () -> new BlueprintServiceConfigEntity());
+ serviceConfigEntities.forEach( configEntity -> configEntity.setService(serviceEntity) );
+ serviceEntity.setConfigurations(serviceConfigEntities);
+ mpackEntity.getServiceInstances().add(serviceEntity);
+ serviceEntity.setMpackInstance(mpackEntity);
+ });
+ return mpackEntity;
+ }
+
+ public static MpackInstance fromEntity(BlueprintMpackInstanceEntity entity) {
+ MpackInstance mpack = new MpackInstance();
+ mpack.setUrl(entity.getMpackUri());
+ mpack.setMpackName(entity.getMpackName());
+ mpack.setMpackVersion(entity.getMpackVersion());
+ mpack.setConfiguration(BlueprintImpl.fromConfigEntities(entity.getConfigurations()));
+ for (BlueprintServiceEntity serviceEntity: entity.getServiceInstances()) {
+ ServiceInstance serviceInstance = new ServiceInstance();
+ serviceInstance.setName(serviceEntity.getName());
+ serviceInstance.setType(serviceEntity.getType());
+ serviceInstance.setMpackInstance(mpack);
+ serviceInstance.setConfiguration(BlueprintImpl.fromConfigEntities(serviceEntity.getConfigurations()));
+ mpack.getServiceInstances().add(serviceInstance);
+ }
+ return mpack;
+ }
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceInstance.java
new file mode 100644
index 0000000..8fcee98
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceInstance.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class ServiceInstance implements Configurable {
+ private String name;
+ private String type;
+ private Configuration configuration = new Configuration();
+ private MpackInstance mpackInstance;
+
+ public ServiceInstance() { }
+
+ public ServiceInstance(String name, String type, Configuration configuration) {
+ this.name = name;
+ this.type = type;
+ this.configuration = configuration;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @JsonIgnore
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @JsonIgnore
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ @JsonIgnore
+ public MpackInstance getMpackInstance() {
+ return mpackInstance;
+ }
+
+ @JsonIgnore
+ void setMpackInstance(MpackInstance mpackInstance) {
+ this.mpackInstance = mpackInstance;
+ }
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index 6da7671..35add3c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -275,7 +275,7 @@ public class TopologyManager {
final ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request);
final String clusterName = request.getClusterName();
- final Stack stack = topology.getBlueprint().getStack();
+ final Stack stack = topology.getBlueprint().getStack(); // TODO: implement multi-stack
final String repoVersion = request.getRepositoryVersion();
final Long repoVersionID = request.getRepositoryVersionId();
@@ -1095,7 +1095,7 @@ public class TopologyManager {
*/
private void addKerberosClient(ClusterTopology topology) {
for (HostGroup group : topology.getBlueprint().getHostGroups().values()) {
- group.addComponent("KERBEROS_CLIENT");
+ group.addComponent(new Component("KERBEROS_CLIENT"));
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
index 990aee7..74fb9b1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
@@ -52,7 +52,7 @@ public class PersistHostResourcesTask extends TopologyHostTask {
HostGroup group = hostRequest.getHostGroup();
Map<String, Collection<String>> serviceComponents = new HashMap<>();
for (String service : group.getServices()) {
- serviceComponents.put(service, new HashSet<>(group.getComponents(service)));
+ serviceComponents.put(service, new HashSet<>(group.getComponentNames(service)));
}
clusterTopology.getAmbariContext().createAmbariHostResources(hostRequest.getClusterId(),
hostRequest.getHostName(), serviceComponents);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
index 80b2593..9d9ad00 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
@@ -59,7 +59,7 @@ public class HiveServiceValidator implements TopologyValidator {
}
// hive database settings need the mysql-server component in the blueprint
- if (!topology.getBlueprint().getComponents(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) {
+ if (!topology.getBlueprint().getComponentNames(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) {
String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " +
"is configured with the current settings. HIVE service validation failed.", MYSQL_SERVER_COMPONENT);
LOGGER.error(errorMessage);
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index 3d04e48..c45da45 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -618,7 +618,6 @@ CREATE TABLE blueprint_mpack_instance(
CONSTRAINT FK_mpi_blueprint_name FOREIGN KEY (blueprint_name) REFERENCES blueprint(blueprint_name),
CONSTRAINT FK_mpi_mpack_id FOREIGN KEY (mpack_id) REFERENCES mpacks(id));
-
CREATE TABLE blueprint_service (
id BIGINT NOT NULL,
mpack_instance_id BIGINT NOT NULL,
diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json
index 0457a47..aac2140 100644
--- a/ambari-server/src/main/resources/properties.json
+++ b/ambari-server/src/main/resources/properties.json
@@ -379,7 +379,8 @@
"host_groups/cardinality",
"configurations",
"validate_topology",
- "settings"
+ "settings",
+ "mpack_instances"
],
"Recommendation":[
"Recommendation/id",
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 3f6de7d..d9d674d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -156,6 +156,35 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
Collections.emptyMap()).anyTimes();
expect(serviceInfo.getRequiredServices()).andReturn(Collections.emptyList()).anyTimes();
+ setupGetServiceForComponentExpectations();
+
+ expect(stack.getCardinality("MYSQL_SERVER")).andReturn(new Cardinality("0-1")).anyTimes();
+
+ Set<String> emptySet = Collections.emptySet();
+ expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes();
+
+ expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes();
+ expect(configHelper.getDefaultStackProperties(
+ EasyMock.eq(new StackId(STACK_NAME, STACK_VERSION)))).andReturn(stackProperties).anyTimes();
+
+ stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties);
+
+
+ expect(ambariContext.isClusterKerberosEnabled(1)).andReturn(true).once();
+ expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes();
+ PowerMock.mockStatic(AmbariServer.class);
+ expect(AmbariServer.getController()).andReturn(controller).anyTimes();
+ PowerMock.replay(AmbariServer.class);
+ expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes();
+ expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(kerberosHelper.getKerberosDescriptor(cluster, false)).andReturn(kerberosDescriptor).anyTimes();
+ Set<String> properties = new HashSet<>();
+ properties.add("core-site/hadoop.security.auth_to_local");
+ expect(kerberosDescriptor.getAllAuthToLocalProperties()).andReturn(properties).anyTimes();
+ }
+
+ private void setupGetServiceForComponentExpectations() {
Collection<String> hdfsComponents = new HashSet<>();
hdfsComponents.add("NAMENODE");
hdfsComponents.add("SECONDARY_NAMENODE");
@@ -163,12 +192,8 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
hdfsComponents.add("HDFS_CLIENT");
serviceComponents.put("HDFS", hdfsComponents);
- Collection<String> yarnComponents = new HashSet<>();
- yarnComponents.add("RESOURCEMANAGER");
- yarnComponents.add("NODEMANAGER");
- yarnComponents.add("YARN_CLIENT");
- yarnComponents.add("APP_TIMELINE_SERVER");
- serviceComponents.put("YARN", yarnComponents);
+ serviceComponents.put("YARN", Sets.newHashSet("RESOURCEMANAGER", "NODEMANAGER", "YARN_CLIENT",
+ "APP_TIMELINE_SERVER", "HISTORYSERVER"));
Collection<String> mrComponents = new HashSet<>();
mrComponents.add("MAPREDUCE2_CLIENT");
@@ -180,11 +205,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
zkComponents.add("ZOOKEEPER_CLIENT");
serviceComponents.put("ZOOKEEPER", zkComponents);
- Collection<String> hiveComponents = new HashSet<>();
- hiveComponents.add("MYSQL_SERVER");
- hiveComponents.add("HIVE_METASTORE");
- hiveComponents.add("HIVE_SERVER");
- serviceComponents.put("HIVE", hiveComponents);
+ serviceComponents.put("HIVE", Sets.newHashSet("MYSQL_SERVER", "HIVE_METASTORE", "HIVE_SERVER", "HIVE_CLIENT"));
Collection<String> falconComponents = new HashSet<>();
falconComponents.add("FALCON_SERVER");
@@ -222,9 +243,11 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
amsComponents.add("METRICS_COLLECTOR");
serviceComponents.put("AMBARI_METRICS", amsComponents);
- Collection<String> stormComponents = new HashSet<>();
- stormComponents.add("NIMBUS");
- serviceComponents.put("STORM", stormComponents);
+ serviceComponents.put("STORM", Sets.newHashSet("NIMBUS", "SUPERVISOR", "STORM_UI_SERVER", "DRPC_SERVER"));
+ serviceComponents.put("RANGER", Sets.newHashSet("RANGER_ADMIN", "RANGER_USERSYNC", "RANGER_KMS_SERVER"));
+ serviceComponents.put("DRUID", Sets.newHashSet("DRUID_COORDINATOR", "DRUID_OVERLORD", "DRUID_BROKER", "DRUID_ROUTER"));
+ serviceComponents.put("HAWQ", Sets.newHashSet("HAWQMASTER", "HAWQSTANDBY"));
+ serviceComponents.put("TEZ", Sets.newHashSet("TEZ_CLIENT"));
for (Map.Entry<String, Collection<String>> entry : serviceComponents.entrySet()) {
String service = entry.getKey();
@@ -232,31 +255,6 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(stack.getServiceForComponent(component)).andReturn(service).anyTimes();
}
}
-
- expect(stack.getCardinality("MYSQL_SERVER")).andReturn(new Cardinality("0-1")).anyTimes();
-
- Set<String> emptySet = Collections.emptySet();
- expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes();
-
- expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes();
- expect(configHelper.getDefaultStackProperties(
- EasyMock.eq(new StackId(STACK_NAME, STACK_VERSION)))).andReturn(stackProperties).anyTimes();
-
- stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties);
-
-
- expect(ambariContext.isClusterKerberosEnabled(1)).andReturn(true).once();
- expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes();
- PowerMock.mockStatic(AmbariServer.class);
- expect(AmbariServer.getController()).andReturn(controller).anyTimes();
- PowerMock.replay(AmbariServer.class);
- expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes();
- expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes();
- expect(controller.getClusters()).andReturn(clusters).anyTimes();
- expect(kerberosHelper.getKerberosDescriptor(cluster, false)).andReturn(kerberosDescriptor).anyTimes();
- Set<String> properties = new HashSet<>();
- properties.add("core-site/hadoop.security.auth_to_local");
- expect(kerberosDescriptor.getAllAuthToLocalProperties()).andReturn(properties).anyTimes();
}
@After
@@ -4275,6 +4273,10 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(stack.getExcludedConfigurationTypes("OOZIE")).andReturn(Collections.emptySet());
expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes();
expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes();
+ expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE");
+ expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE");
Map<String, Map<String, String>> properties = new HashMap<>();
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -4314,6 +4316,10 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes();
expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes();
expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes();
+ expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE");
+ expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE");
Map<String, Map<String, String>> properties = new HashMap<>();
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -4350,6 +4356,8 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// customized stack calls for this test only
expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes();
expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes();
+ expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON");
Map<String, Map<String, String>> properties = new HashMap<>();
Map<String, String> typeProps = new HashMap<>();
@@ -4397,6 +4405,10 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes();
// simulate the case where the STORM service has been removed manually from the stack definitions
expect(stack.getServiceForConfigType("storm-site")).andThrow(new IllegalArgumentException("TEST: Configuration not found in stack definitions!"));
+ expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON");
+ expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE");
+ expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE");
Map<String, Map<String, String>> properties = new HashMap<>();
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -4593,6 +4605,8 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(stack.getServiceForConfigType("hive-site")).andReturn("HIVE").atLeastOnce();
expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce();
+ expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS").anyTimes();
+
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
Collection<String> hgComponents = new HashSet<>();
@@ -4670,6 +4684,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// simulate the case of the stack object throwing a RuntimeException, to indicate a config error
expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("Expected Test Error")).once();
expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce();
+ expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS");
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -4748,6 +4763,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// customized stack calls for this test only
expect(stack.getServiceForConfigType("hive-site")).andReturn("HIVE").atLeastOnce();
expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce();
+ expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS");
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -4836,6 +4852,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// customized stack calls for this test only
expect(stack.getServiceForConfigType("hbase-site")).andReturn("HBASE").atLeastOnce();
expect(stack.getConfigurationPropertiesWithMetadata("HBASE", "hbase-site")).andReturn(mapOfMetadata).atLeastOnce();
+ expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS");
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -4904,6 +4921,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// customized stack calls for this test only
expect(stack.getServiceForConfigType("hbase-site")).andReturn("HBASE").atLeastOnce();
expect(stack.getConfigurationPropertiesWithMetadata("HBASE", "hbase-site")).andReturn(mapOfMetadata).atLeastOnce();
+ expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS");
Configuration clusterConfig = new Configuration(properties, Collections.emptyMap());
@@ -8214,7 +8232,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
}
//create host group which is set on topology
- allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", stack,
+ allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", Collections.singleton(stack),
componentList, EMPTY_CONFIG, "1"));
hostGroupInfo.put(hostGroup.name, groupInfo);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
index fc61a20..e303731 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
@@ -109,7 +109,7 @@ public class BlueprintResourceProviderTest {
@BeforeClass
public static void initClass() {
- BlueprintResourceProvider.init(blueprintFactory, dao, securityFactory, gson, metaInfo);
+ BlueprintResourceProvider.init(blueprintFactory, dao, securityFactory, metaInfo);
}
private Map<String, Set<HashMap<String, String>>> getSettingProperties() {
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
index e1f5583..e70901a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
@@ -20,6 +20,7 @@
package org.apache.ambari.server.controller.internal;
import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
@@ -28,8 +29,11 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
import java.lang.reflect.Field;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -49,11 +53,17 @@ import org.apache.ambari.server.topology.HostGroupInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
/**
* ExportBlueprintRequest unit tests.
*/
@SuppressWarnings("unchecked")
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ExportBlueprintRequest.ExportedHostGroup.class})
public class ExportBlueprintRequestTest {
private static final String CLUSTER_NAME = "c1";
private static final String CLUSTER_ID = "2";
@@ -70,8 +80,12 @@ public class ExportBlueprintRequestTest {
Collections.emptySet()).anyTimes();
expect(controller.getStackLevelConfigurations((Set<StackLevelConfigurationRequest>) anyObject())).andReturn(
Collections.emptySet()).anyTimes();
-
replay(controller);
+
+ // This can save precious time
+ mockStatic(InetAddress.class);
+ expect(InetAddress.getByName(anyString())).andThrow(new UnknownHostException()).anyTimes();
+ PowerMock.replay(InetAddress.class);
}
@After
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java
index 5bbf75e..0e4f61a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java
@@ -34,7 +34,6 @@ import org.apache.ambari.server.controller.internal.ProvisionAction;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
import org.apache.ambari.server.orm.dao.BlueprintDAO;
-import org.apache.ambari.server.orm.dao.StackDAO;
import org.apache.ambari.server.state.SecurityType;
import org.junit.After;
import org.junit.Before;
@@ -54,8 +53,6 @@ public class BlueprintEntityTest2 {
@Inject
private Injector injector;
@Inject
- private StackDAO stackDAO;
- @Inject
private BlueprintDAO blueprintDAO;
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java b/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java
index f83c46a..312c905 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java
@@ -30,7 +30,7 @@ import org.junit.rules.TemporaryFolder;
import junit.framework.Assert;
-public class CredentialStoreTest {
+public class CredentialStoreTest {
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
index 6ee1c5d..1bd79e1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
@@ -243,8 +243,8 @@ public class AmbariContextTest {
expect(blueprint.getName()).andReturn(BP_NAME).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(blueprint.getServices()).andReturn(blueprintServices).anyTimes();
- expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes();
- expect(blueprint.getComponents("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes();
+ expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes();
+ expect(blueprint.getComponentNames("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes();
expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes();
expect(blueprint.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java
index e7c0fe9..c2bf712 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java
@@ -18,13 +18,18 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toSet;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.powermock.api.easymock.PowerMock.createStrictMock;
+import static org.powermock.api.easymock.PowerMock.expectNew;
import static org.powermock.api.easymock.PowerMock.replay;
import static org.powermock.api.easymock.PowerMock.reset;
import static org.powermock.api.easymock.PowerMock.verify;
@@ -35,8 +40,10 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.ambari.server.ObjectNotFoundException;
+import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
import org.apache.ambari.server.controller.internal.BlueprintResourceProviderTest;
import org.apache.ambari.server.controller.internal.Stack;
@@ -48,11 +55,20 @@ import org.easymock.EasyMockSupport;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
/**
* BlueprintFactory unit tests.
*/
@SuppressWarnings("unchecked")
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(BlueprintImpl.class)
public class BlueprintFactoryTest {
private static final String BLUEPRINT_NAME = "test-blueprint";
@@ -108,6 +124,42 @@ public class BlueprintFactoryTest {
// }
@Test
+ public void testGetMultiInstanceBlueprint() throws Exception {
+ // prepare
+ Blueprint expectedBlueprint = createMultiInstanceBlueprint();
+
+ reset(dao);
+ expect(dao.findByName(BLUEPRINT_NAME)).andReturn(expectedBlueprint.toEntity());
+ Stack hdpStack = createNiceMock(Stack.class);
+ expect(hdpStack.getName()).andReturn("HDPCORE").anyTimes();
+ expect(hdpStack.getVersion()).andReturn("3.0.0.0").anyTimes();
+ Stack edwStack = createNiceMock(Stack.class);
+ expect(edwStack.getName()).andReturn("EDW").anyTimes();
+ expect(edwStack.getVersion()).andReturn("3.1.0.0").anyTimes();
+ expectNew(Stack.class, eq("HDPCORE-3.0"), anyString(), anyObject(AmbariManagementController.class)).andReturn(hdpStack).anyTimes();
+ expectNew(Stack.class, eq("EDW-3.1"), anyString(), anyObject(AmbariManagementController.class)).andReturn(edwStack).anyTimes();
+ replay(Stack.class, hdpStack, edwStack, dao);
+
+ // test
+ Blueprint blueprint = testFactory.getBlueprint(BLUEPRINT_NAME);
+ Set<String> mpackNames =
+ blueprint.getMpacks().stream().map(MpackInstance::getMpackName).collect(Collectors.toSet());
+ assertEquals(ImmutableSet.of("HDPCORE-3.0", "EDW-3.1"), mpackNames );
+ MpackInstance hdpCore =
+ blueprint.getMpacks().stream().filter(mp -> "HDPCORE-3.0".equals(mp.getMpackName())).findAny().get();
+ Set<String> serviceInstanceNames =
+ hdpCore.getServiceInstances().stream().map(ServiceInstance::getName).collect(toSet());
+ assertEquals(ImmutableSet.of("ZK1", "ZK2"), serviceInstanceNames);
+ Set<String> serviceInstanceTypes =
+ hdpCore.getServiceInstances().stream().map(ServiceInstance::getType).collect(toSet());
+ assertEquals(ImmutableSet.of("ZOOKEEPER"), serviceInstanceTypes);
+ Set<String> stackNames =
+ blueprint.getStacks().stream().map(Stack::getName).collect(Collectors.toSet());
+ assertEquals(ImmutableSet.of("HDPCORE", "EDW"), stackNames);
+ assertEquals(1, blueprint.getHostGroups().size());
+ }
+
+ @Test
public void testGetBlueprint_NotFound() throws Exception {
expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null).once();
replay(dao, entity, configEntity);
@@ -163,6 +215,43 @@ public class BlueprintFactoryTest {
verify(dao, entity, configEntity);
}
+ @Test
+ public void testCreateMultiInstanceBlueprint() throws Exception {
+ createMultiInstanceBlueprint();
+ }
+
+ public Blueprint createMultiInstanceBlueprint() throws Exception {
+ Map<String, Collection<String>> allComponents = ImmutableMap.of(
+ "HDFS", ImmutableSet.of("NAMENODE", "SECONDARY_NAMENODE"),
+ "ZOOKEEPER", ImmutableSet.of("ZOOKEEPER_SERVER")
+ );
+ reset(stack);
+ expect(stack.getComponents()).andReturn(allComponents).anyTimes();
+ expect(stack.isMasterComponent("NAMENODE")).andReturn(true).anyTimes();
+ expect(stack.isMasterComponent("ZOOKEEPER_SERVER")).andReturn(true).anyTimes();
+ expect(stack.isMasterComponent("SECONDAY_NAMENODE")).andReturn(false).anyTimes();
+ expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS").anyTimes();
+ expect(stack.getServiceForComponent("SECONDARY_NAMENODE")).andReturn("HDFS").anyTimes();
+ expect(stack.getServiceForComponent("ZOOKEEPER_SERVER")).andReturn("ZOOKEEPER").anyTimes();
+
+ replay(stack, dao, entity, configEntity);
+
+ Map<String, Object> props = BlueprintTestUtil.getMultiInstanceBlueprintAsMap();
+ Blueprint blueprint = testFactory.createBlueprint(props, null);
+
+ assertEquals(2, blueprint.getMpacks().size());
+ assertEquals(Sets.newHashSet("HDPCORE-3.0", "EDW-3.1"),
+ blueprint.getMpacks().stream().map(MpackInstance::getMpackName).collect(toSet()));
+ MpackInstance hdpCore =
+ blueprint.getMpacks().stream().filter( mpack -> "HDPCORE-3.0".equals(mpack.getMpackName()) ).findFirst().get();
+ assertEquals(2, hdpCore.getServiceInstances().size());
+ ServiceInstance zk1 =
+ hdpCore.getServiceInstances().stream().filter( si -> "ZK1".equals(si.getName()) ).findFirst().get();
+ assertEquals("ZOOKEEPER", zk1.getType());
+ assertEquals("/zookeeper1", zk1.getConfiguration().getProperties().get("zoo.cfg").get("dataDir"));
+ return blueprint;
+ }
+
@Test(expected=NoSuchStackException.class)
public void testCreateInvalidStack() throws Exception {
EasyMockSupport mockSupport = new EasyMockSupport();
@@ -240,7 +329,7 @@ public class BlueprintFactoryTest {
}
@Override
- protected Stack createStack(Map<String, Object> properties) throws NoSuchStackException {
+ protected Stack loadStack(String stackName, String stackVersion) throws NoSuchStackException {
return stack;
}
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index d34526a..057332d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -124,7 +124,7 @@ public class BlueprintImplTest {
category2Props.put("prop2", "val");
SecurityConfiguration securityConfiguration = new SecurityConfiguration(SecurityType.KERBEROS, "testRef", null);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, securityConfiguration);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, securityConfiguration, null);
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
@@ -162,7 +162,7 @@ public class BlueprintImplTest {
hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP:group1%");
hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP:group2%");
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null);
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
verify(stack, group1, group2, serverConfig);
@@ -202,7 +202,7 @@ public class BlueprintImplTest {
hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%");
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null);
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
@@ -243,7 +243,7 @@ public class BlueprintImplTest {
hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%");
hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group3%");
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null);
blueprint.validateRequiredProperties();
verify(stack, group1, group2, serverConfig);
}
@@ -279,7 +279,7 @@ public class BlueprintImplTest {
hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%");
hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%");
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null);
blueprint.validateRequiredProperties();
verify(stack, group1, group2, serverConfig);
}
@@ -294,7 +294,7 @@ public class BlueprintImplTest {
hdfsProps.put("secret", "SECRET:hdfs-site:1:test");
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null);
blueprint.validateRequiredProperties();
verify(stack, group1, group2, serverConfig);
}
@@ -311,7 +311,7 @@ public class BlueprintImplTest {
serverConfig = setupConfigurationWithGPLLicense(false);
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null, null);
blueprint.validateRequiredProperties();
verify(stack, group1, group2, serverConfig);
}
@@ -328,7 +328,7 @@ public class BlueprintImplTest {
serverConfig = setupConfigurationWithGPLLicense(false);
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null, null);
blueprint.validateRequiredProperties();
verify(stack, group1, group2, serverConfig);
}
@@ -346,32 +346,36 @@ public class BlueprintImplTest {
expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
replay(stack, group1, group2, serverConfig);
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null, null);
blueprint.validateRequiredProperties();
verify(stack, group1, group2, serverConfig);
}
@Test
public void testAutoSkipFailureEnabled() {
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting);
HashMap<String, String> skipFailureSetting = new HashMap<>();
skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "true");
+ expect(stack.getName()).andReturn("HDPCORE").anyTimes();
+ expect(stack.getVersion()).andReturn("3.0.0.0").anyTimes();
expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting));
replay(stack, setting);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting);
assertTrue(blueprint.shouldSkipFailure());
+
verify(stack, setting);
}
@Test
public void testAutoSkipFailureDisabled() {
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting);
HashMap<String, String> skipFailureSetting = new HashMap<>();
skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "false");
expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting));
replay(stack, setting);
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting);
assertFalse(blueprint.shouldSkipFailure());
+
verify(stack, setting);
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintTestUtil.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintTestUtil.java
new file mode 100644
index 0000000..1bc3723
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintTestUtil.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.io.Resources;
+
+public class BlueprintTestUtil {
+ public static final String BLUEPRINT_30_LOCATION = "blueprint3.0/multi-instance-blueprint.json";
+
+ public static Map<String, Object> getMultiInstanceBlueprintAsMap() {
+ URL url = Resources.getResource(BLUEPRINT_30_LOCATION);
+ try {
+ Map<String, Object> blueprintMap = new ObjectMapper().readValue(url, new TypeReference<Map<String, Object>>(){});
+
+ Map<String, Object> blueprintProps = (Map<String, Object>) blueprintMap.remove("Blueprints");
+ blueprintProps.forEach( (key, value) -> blueprintMap.put("Blueprints/" + key, value));
+ Map<String, Object> settingProps = (Map<String, Object>) blueprintMap.remove("settings");
+ settingProps.forEach( (key, value) -> blueprintMap.put("settings/" + key, value));
+
+ return convertListsToSets(blueprintMap);
+ }
+ catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ private static Map<String, Object> convertListsToSets(Map<String, Object> input) {
+ Map<String, Object> converted = new HashMap<>();
+ input.forEach( (k, v) -> {
+ if (v instanceof List) {
+ List<Object> list = (List<Object>)v;
+ Set<Object> set = list.stream().map(e -> {
+ if (e instanceof Map) {
+ return convertListsToSets((Map<String, Object>)e);
+ }
+ else {
+ return e;
+ }
+ }).collect(Collectors.toSet());
+ converted.put(k, set);
+ }
+ else if (v instanceof Map){
+ converted.put(k, convertListsToSets((Map<String, Object>)v));
+ }
+ else {
+ converted.put(k, v);
+ }
+ });
+ return converted;
+ }
+
+
+}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index 75a9d6b..e88bef3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toList;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
@@ -103,8 +104,12 @@ public class BlueprintValidatorImplTest {
expect(blueprint.getServices()).andReturn(services).anyTimes();
expect(group1.getComponentNames()).andReturn(group1Components).anyTimes();
+ expect(group1.getComponents()).
+ andAnswer(() -> group1Components.stream().map(comp -> new Component(comp)).collect(toList())).anyTimes();
expect(group1.getName()).andReturn("host-group-1").anyTimes();
expect(group2.getComponentNames()).andReturn(group2Components).anyTimes();
+ expect(group2.getComponents()).
+ andAnswer(() -> group2Components.stream().map(comp -> new Component(comp)).collect(toList())).anyTimes();
expect(group2.getName()).andReturn("host-group-2").anyTimes();
expect(stack.getDependenciesForComponent("component1")).andReturn(dependencies1).anyTimes();
@@ -119,6 +124,7 @@ public class BlueprintValidatorImplTest {
dependenciesConditionInfos1.add(dependencyConditionInfo2);
expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(blueprint.isAllMpacksResolved()).andReturn(true);
}
@After
@@ -171,7 +177,7 @@ public class BlueprintValidatorImplTest {
expect(stack.getComponents("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes();
expect(stack.getAutoDeployInfo("component1")).andReturn(autoDeploy).anyTimes();
- expect(group1.addComponent("component1")).andReturn(true).once();
+ expect(group1.addComponent(new Component("component1"))).andReturn(true).once();
replay(blueprint, stack, group1, group2, dependency1);
BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
@@ -207,8 +213,8 @@ public class BlueprintValidatorImplTest {
expect(dependencyComponentInfo.isClient()).andReturn(true).anyTimes();
expect(stack.getComponentInfo("component3")).andReturn(dependencyComponentInfo).anyTimes();
- expect(group1.addComponent("component1")).andReturn(true).once();
- expect(group1.addComponent("component3")).andReturn(true).once();
+ expect(group1.addComponent(new Component("component1"))).andReturn(true).once();
+ expect(group1.addComponent(new Component("component3"))).andReturn(true).once();
replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index 771b89f..28ce72a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -257,9 +257,9 @@ public class ClusterConfigurationRequestTest {
List<String> zookeeperComponents = new ArrayList<>();
zookeeperComponents.add("ZOOKEEPER_SERVER");
- expect(blueprint.getComponents("HDFS")).andReturn(hdfsComponents).anyTimes();
- expect(blueprint.getComponents("KERBEROS")).andReturn(kerberosComponents).anyTimes();
- expect(blueprint.getComponents("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes();
+ expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes();
+ expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes();
+ expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes();
expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes();
expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes();
@@ -346,9 +346,9 @@ public class ClusterConfigurationRequestTest {
List<String> zookeeperComponents = new ArrayList<>();
zookeeperComponents.add("ZOOKEEPER_SERVER");
- expect(blueprint.getComponents("HDFS")).andReturn(hdfsComponents).anyTimes();
- expect(blueprint.getComponents("KERBEROS")).andReturn(kerberosComponents).anyTimes();
- expect(blueprint.getComponents("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes();
+ expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes();
+ expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes();
+ expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes();
expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes();
expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java
index aecc6cb..d0e82f2 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java
@@ -241,8 +241,8 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes();
expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes();
- expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
- expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
+ expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
+ expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes();
expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes();
expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes();
@@ -304,8 +304,8 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(group1.containsMasterComponent()).andReturn(true).anyTimes();
expect(group1.getComponentNames()).andReturn(group1Components).anyTimes();
expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes();
- expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
- expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
+ expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
+ expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes();
expect(group1.getName()).andReturn("group1").anyTimes();
expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes();
@@ -316,8 +316,8 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(group2.containsMasterComponent()).andReturn(false).anyTimes();
expect(group2.getComponentNames()).andReturn(group2Components).anyTimes();
expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes();
- expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes();
- expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes();
+ expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes();
+ expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes();
expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes();
expect(group2.getName()).andReturn("group2").anyTimes();
expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java
index a4b2160..c64e228 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toList;
import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START;
import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyLong;
@@ -182,10 +183,11 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
private Collection<String> group1Components = Arrays.asList("component1", "component2", "component3");
private Collection<String> group2Components = Arrays.asList("component3", "component4");
- private Map<String, Collection<String>> group1ServiceComponents = new HashMap<>();
- private Map<String, Collection<String>> group2ServiceComponents = new HashMap<>();
+ private Map<String, Collection<Component>> group1ServiceComponents = new HashMap<>();
+ private Map<String, Collection<Component>> group2ServiceComponents = new HashMap<>();
- private Map<String, Collection<String>> serviceComponents = new HashMap<>();
+ private Map<String, Collection<Component>> serviceComponents = new HashMap<>();
+ private Map<String, Collection<String>> serviceComponentNames = new HashMap<>();
private String predicate = "Hosts/host_name=foo";
@@ -198,6 +200,7 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
private Capture<ClusterRequest> updateClusterConfigRequestCapture;
private Capture<Runnable> updateConfigTaskCapture;
+
@Before
public void setup() throws Exception {
clusterTopologyCapture = newCapture();
@@ -227,18 +230,22 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
groupMap.put("group1", group1);
groupMap.put("group2", group2);
- serviceComponents.put("service1", Arrays.asList("component1", "component3"));
- serviceComponents.put("service2", Arrays.asList("component2", "component4"));
+ serviceComponents.put("service1", Arrays.asList(new Component("component1"), new Component("component3")));
+ serviceComponents.put("service2", Arrays.asList(new Component("component2"), new Component("component4")));
+
+ for(Map.Entry<String, Collection<Component>> entry: serviceComponents.entrySet()) {
+ serviceComponentNames.put(entry.getKey(), entry.getValue().stream().map(Component::getName).collect(toList()));
+ }
- group1ServiceComponents.put("service1", Arrays.asList("component1", "component3"));
- group1ServiceComponents.put("service2", Collections.singleton("component2"));
- group2ServiceComponents.put("service2", Collections.singleton("component3"));
- group2ServiceComponents.put("service2", Collections.singleton("component4"));
+ group1ServiceComponents.put("service1", Arrays.asList(new Component("component1"), new Component("component3")));
+ group1ServiceComponents.put("service2", Collections.singleton(new Component("component2")));
+ group2ServiceComponents.put("service2", Collections.singleton(new Component("component3")));
+ group2ServiceComponents.put("service2", Collections.singleton(new Component("component4")));
expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes();
expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes();
- expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
- expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
+ expect(blueprint.getComponents("service1")).andReturn(Arrays.asList(new Component("component1"), new Component("component3"))).anyTimes();
+ expect(blueprint.getComponents("service2")).andReturn(Arrays.asList(new Component("component2"), new Component("component4"))).anyTimes();
expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes();
expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes();
expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes();
@@ -273,9 +280,9 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
expect(stack.getCardinality("component2")).andReturn(new Cardinality("1")).anyTimes();
expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+")).anyTimes();
expect(stack.getCardinality("component4")).andReturn(new Cardinality("1+")).anyTimes();
- expect(stack.getComponents()).andReturn(serviceComponents).anyTimes();
- expect(stack.getComponents("service1")).andReturn(serviceComponents.get("service1")).anyTimes();
- expect(stack.getComponents("service2")).andReturn(serviceComponents.get("service2")).anyTimes();
+ expect(stack.getComponents()).andReturn(serviceComponentNames).anyTimes();
+ expect(stack.getComponents("service1")).andReturn(serviceComponentNames.get("service1")).anyTimes();
+ expect(stack.getComponents("service2")).andReturn(serviceComponentNames.get("service2")).anyTimes();
expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
index d89c8ca..6fa70b4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
@@ -239,8 +239,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes();
expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes();
- expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
- expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
+ expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
+ expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes();
expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes();
expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes();
@@ -303,8 +303,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
expect(group1.getComponentNames()).andReturn(group1Components).anyTimes();
expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes();
- expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
- expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
+ expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
+ expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes();
expect(group1.getName()).andReturn("group1").anyTimes();
expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes();
@@ -315,8 +315,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
expect(group2.containsMasterComponent()).andReturn(false).anyTimes();
expect(group2.getComponentNames()).andReturn(group2Components).anyTimes();
expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes();
- expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes();
- expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes();
+ expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes();
+ expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes();
expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes();
expect(group2.getName()).andReturn("group2").anyTimes();
expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java
new file mode 100644
index 0000000..dfcee84
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import static org.junit.Assert.assertEquals;
+
+import java.net.URL;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Resources;
+
+public class ConfigurableTest {
+ public static final String JSON_LOCATION = "blueprint3.0/configurable.json";
+ public static final String JSON_LOCATION2 = "blueprint3.0/configurable2.json";
+
+ private TestConfigurable configurable;
+ private ObjectMapper mapper;
+
+ @Before
+ public void setUp() throws Exception {
+ mapper = new ObjectMapper();
+ URL url = Resources.getResource(JSON_LOCATION);
+ configurable = new ObjectMapper().readValue(url, TestConfigurable.class);
+ }
+
+ @Test
+ public void testParseConfigurable() throws Exception{
+ assertEquals(ImmutableMap.of("zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1")),
+ configurable.getConfiguration().getProperties());
+ assertEquals(
+ ImmutableMap.of("zoo.cfg",
+ ImmutableMap.of("final",
+ ImmutableMap.of("someProp", "someValue"))),
+ configurable.getConfiguration().getAttributes());
+ }
+
+ @Test
+ public void testSerializaDeserialize() throws Exception {
+ String persisted = mapper.writeValueAsString(configurable);
+ Configurable restored = mapper.readValue(persisted, TestConfigurable.class);
+ assertEquals(configurable.getConfiguration().getProperties(), restored.getConfiguration().getProperties());
+ assertEquals(configurable.getConfiguration().getAttributes(), restored.getConfiguration().getAttributes());
+ }
+
+ @Test
+ public void testParseConfigurableFromResoueceManager() throws Exception{
+ mapper = new ObjectMapper();
+ URL url = Resources.getResource(JSON_LOCATION2);
+ configurable = new ObjectMapper().readValue(url, TestConfigurable.class);
+
+ assertEquals(ImmutableMap.of("zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1")),
+ configurable.getConfiguration().getProperties());
+ assertEquals(
+ ImmutableMap.of("zoo.cfg",
+ ImmutableMap.of("final",
+ ImmutableMap.of("someProp", "someValue"))),
+ configurable.getConfiguration().getAttributes());
+ }
+
+
+}
+
+class TestConfigurable implements Configurable {
+ Configuration configuration;
+
+ @Override
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+
+}
\ No newline at end of file
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
index c474493..722f5f1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
@@ -149,12 +149,12 @@ public class RequiredPasswordValidatorTest extends EasyMockSupport {
expect(group1.getComponentNames()).andReturn(group1Components).anyTimes();
expect(group2.getComponentNames()).andReturn(group2Components).anyTimes();
- expect(group1.getComponents("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes();
- expect(group1.getComponents("service2")).andReturn(Arrays.asList("component3")).anyTimes();
- expect(group1.getComponents("service3")).andReturn(Collections.emptySet()).anyTimes();
- expect(group2.getComponents("service1")).andReturn(Arrays.asList("component1")).anyTimes();
- expect(group2.getComponents("service2")).andReturn(Collections.emptySet()).anyTimes();
- expect(group2.getComponents("service3")).andReturn(Arrays.asList("component4")).anyTimes();
+ expect(group1.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes();
+ expect(group1.getComponentNames("service2")).andReturn(Arrays.asList("component3")).anyTimes();
+ expect(group1.getComponentNames("service3")).andReturn(Collections.emptySet()).anyTimes();
+ expect(group2.getComponentNames("service1")).andReturn(Arrays.asList("component1")).anyTimes();
+ expect(group2.getComponentNames("service2")).andReturn(Collections.emptySet()).anyTimes();
+ expect(group2.getComponentNames("service3")).andReturn(Arrays.asList("component4")).anyTimes();
expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes();
expect(stack.getServiceForComponent("component2")).andReturn("service1").anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
index c12965d..0daf20f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
@@ -244,8 +244,8 @@ public class TopologyManagerTest {
expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes();
expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes();
- expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
- expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
+ expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes();
+ expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes();
expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes();
expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes();
expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes();
@@ -295,8 +295,8 @@ public class TopologyManagerTest {
expect(group1.getCardinality()).andReturn("test cardinality").anyTimes();
expect(group1.containsMasterComponent()).andReturn(true).anyTimes();
expect(group1.getComponents()).andReturn(group1Components).anyTimes();
- expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
- expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
+ expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
+ expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes();
expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes();
expect(group1.getName()).andReturn("group1").anyTimes();
expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes();
@@ -306,8 +306,8 @@ public class TopologyManagerTest {
expect(group2.getCardinality()).andReturn("test cardinality").anyTimes();
expect(group2.containsMasterComponent()).andReturn(false).anyTimes();
expect(group2.getComponents()).andReturn(group2Components).anyTimes();
- expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes();
- expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes();
+ expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes();
+ expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes();
expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes();
expect(group2.getName()).andReturn("group2").anyTimes();
expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes();
@@ -407,8 +407,8 @@ public class TopologyManagerTest {
expect(persistedState.getAllRequests()).andReturn(allRequests).anyTimes();
expect(persistedState.getProvisionRequest(CLUSTER_ID)).andReturn(logicalRequest).anyTimes();
expect(ambariContext.isTopologyResolved(CLUSTER_ID)).andReturn(true).anyTimes();
- expect(group1.addComponent("KERBEROS_CLIENT")).andReturn(true).anyTimes();
- expect(group2.addComponent("KERBEROS_CLIENT")).andReturn(true).anyTimes();
+ expect(group1.addComponent(new Component("KERBEROS_CLIENT"))).andReturn(true).anyTimes();
+ expect(group2.addComponent(new Component("KERBEROS_CLIENT"))).andReturn(true).anyTimes();
replayAll();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java
index f4c5ca5..3ce0a02 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java
@@ -122,7 +122,7 @@ public class HiveServiceValidatorTest extends EasyMockSupport {
Collection<String> configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env");
EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes();
EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes();
- EasyMock.expect(blueprintMock.getComponents("HIVE")).andReturn(Collections.emptyList()).anyTimes();
+ EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(Collections.emptyList()).anyTimes();
EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock);
EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes);
@@ -145,7 +145,7 @@ public class HiveServiceValidatorTest extends EasyMockSupport {
Collection<String> configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env");
EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes();
EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes();
- EasyMock.expect(blueprintMock.getComponents("HIVE")).andReturn(hiveComponents).anyTimes();
+ EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(hiveComponents).anyTimes();
EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock);
EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes);
diff --git a/ambari-server/src/test/resources/blueprint3.0/configurable.json b/ambari-server/src/test/resources/blueprint3.0/configurable.json
new file mode 100644
index 0000000..9bac290
--- /dev/null
+++ b/ambari-server/src/test/resources/blueprint3.0/configurable.json
@@ -0,0 +1,16 @@
+{
+ "configurations" : [
+ {
+ "zoo.cfg" : {
+ "properties" : {
+ "dataDir" : "/zookeeper1"
+ },
+ "properties_attributes": {
+ "final": {
+ "someProp": "someValue"
+ }
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/ambari-server/src/test/resources/blueprint3.0/configurable2.json b/ambari-server/src/test/resources/blueprint3.0/configurable2.json
new file mode 100644
index 0000000..fb813ed
--- /dev/null
+++ b/ambari-server/src/test/resources/blueprint3.0/configurable2.json
@@ -0,0 +1,8 @@
+{
+ "configurations" : [
+ {
+ "zoo.cfg/properties/dataDir" : "/zookeeper1",
+ "zoo.cfg/properties_attributes/final/someProp": "someValue"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/ambari-server/src/test/resources/blueprint3.0/multi-instance-blueprint.json b/ambari-server/src/test/resources/blueprint3.0/multi-instance-blueprint.json
new file mode 100644
index 0000000..d5d21ae
--- /dev/null
+++ b/ambari-server/src/test/resources/blueprint3.0/multi-instance-blueprint.json
@@ -0,0 +1,104 @@
+{
+ "host_groups" : [
+ {
+ "name" : "master",
+ "components" : [
+ {
+ "name" : "NAMENODE"
+ },
+ {
+ "name" : "ZOOKEEPER_SERVER",
+ "mpack-instance": "HDPCORE-3.0",
+ "service-instance": "ZK1"
+ },
+ {
+ "name" : "SECONDARY_NAMENODE"
+ }
+ ],
+ "cardinality" : "1"
+ }
+ ],
+ "Blueprints" : {
+ "blueprint_name" : "test-blueprint",
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
+ },
+ "settings" : {
+ "deployment_settings": [
+ { "skip_failure":"true" }
+ ],
+ "repository_settings":[
+ {
+ "override_strategy":"ALWAYS_APPLY",
+ "operating_system":"redhat7",
+ "repo_id":"HDP",
+ "base_url":"http://myserver/hdp"
+ },
+ {
+ "override_strategy":"APPLY_WHEN_MISSING",
+ "operating_system":"redhat7",
+ "repo_id":"HDP-UTIL-1.1",
+ "base_url":"http://myserver/hdp-util"
+ }
+ ],
+ "recovery_settings":[
+ { "recovery_enabled":"true" }
+ ],
+ "service_settings":[
+ {
+ "name":"HDFS",
+ "recovery_enabled":"false"
+ }
+ ],
+ "component_settings":[
+ {
+ "name":"SECONDARY_NAMENODE",
+ "recovery_enabled" : "true"
+ }
+ ]
+ },
+ "configurations": [],
+
+ "mpack_instances" : [
+ {
+ "name" : "HDPCORE-3.0",
+ "version" : "3.0.0.0",
+ "url" : "http://www.hortonworks.com/path/to/hdpcore.tar.gz",
+ "configurations" : [],
+ "service_instances" : [
+ {
+ "name" : "ZK1",
+ "type" : "ZOOKEEPER",
+ "configurations" : [
+ {
+ "zoo.cfg" : {
+ "properties" : {
+ "dataDir" : "/zookeeper1"
+ }
+ }
+ }
+ ],
+ "settings" : []
+ },
+ {
+ "name" : "ZK2",
+ "type" : "ZOOKEEPER",
+ "configurations" : [
+ {
+ "zoo.cfg" : {
+ "properties" : {
+ "dataDir" : "/zookeeper2"
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name" : "EDW-3.1",
+ "version" : "3.1.0.0",
+ "url" : "http://www.hortonworks.com/path/to/edw.tar.gz"
+ }
+ ]
+}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
benyoka@apache.org.