You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ad...@apache.org on 2018/10/24 13:10:02 UTC
[ambari] branch trunk updated: AMBARI-24709. Implement support for
Minimal Blueprint Export (#2489)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1a8b193 AMBARI-24709. Implement support for Minimal Blueprint Export (#2489)
1a8b193 is described below
commit 1a8b193fbc8da67597044cb26ee1a8cabcc86cd8
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Wed Oct 24 15:09:56 2018 +0200
AMBARI-24709. Implement support for Minimal Blueprint Export (#2489)
---
.../api/query/render/ClusterBlueprintRenderer.java | 61 ++++++--
.../api/resources/ClusterResourceDefinition.java | 11 +-
.../internal/BlueprintConfigurationProcessor.java | 157 +++++++++++++-------
.../controller/internal/BlueprintExportType.java | 164 +++++++++++++++++++++
.../internal/ExportBlueprintRequest.java | 61 +++-----
.../server/controller/internal/UnitUpdater.java | 18 ++-
.../apache/ambari/server/state/PropertyInfo.java | 7 +
.../ambari/server/topology/Configuration.java | 16 +-
.../query/render/ClusterBlueprintRendererTest.java | 43 +++---
.../BlueprintConfigurationProcessorTest.java | 141 +++++++++++++-----
.../internal/ExportBlueprintRequestTest.java | 42 +-----
11 files changed, 516 insertions(+), 205 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index 7ac240a..9139cec 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -41,6 +41,7 @@ import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
+import org.apache.ambari.server.controller.internal.BlueprintExportType;
import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
import org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
import org.apache.ambari.server.controller.internal.RequestImpl;
@@ -72,6 +73,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
/**
* Renderer which renders a cluster resource as a blueprint.
@@ -92,6 +94,12 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
*/
private final AmbariManagementController controller = AmbariServer.getController();
+ private final BlueprintExportType exportType;
+
+ public ClusterBlueprintRenderer(BlueprintExportType exportType) {
+ this.exportType = exportType;
+ }
+
// ----- Renderer ----------------------------------------------------------
@@ -184,7 +192,6 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
* Create a blueprint resource.
*
* @param clusterNode cluster tree node
- *
* @return a new blueprint resource
*/
private Resource createBlueprintResource(TreeNode<Resource> clusterNode) {
@@ -198,7 +205,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
}
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(exportType);
Stack stack = topology.getBlueprint().getStack();
blueprintResource.setProperty("Blueprints/stack_name", stack.getName());
@@ -223,10 +230,16 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
List<Map<String, Object>> groupList = formatGroupsAsList(topology);
blueprintResource.setProperty("host_groups", groupList);
- blueprintResource.setProperty("configurations", processConfigurations(topology));
+ List<Map<String, Map<String, Map<String, ?>>>> configurations = processConfigurations(topology);
+ if (exportType.include(configurations)) {
+ blueprintResource.setProperty("configurations", configurations);
+ }
//Fetch settings section for blueprint
- blueprintResource.setProperty("settings", getSettings(clusterNode, stack));
+ Collection<Map<String, Object>> settings = getSettings(clusterNode, stack);
+ if (exportType.include(settings)) {
+ blueprintResource.setProperty("settings", settings);
+ }
return blueprintResource;
}
@@ -261,7 +274,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
* @return A Collection<Map<String, Object>> which represents the Setting Object
*/
@VisibleForTesting
- static Collection<Map<String, Object>> getSettings(TreeNode<Resource> clusterNode, Stack stack) {
+ Collection<Map<String, Object>> getSettings(TreeNode<Resource> clusterNode, Stack stack) {
//Initialize collections to create appropriate json structure
Collection<Map<String, Object>> blueprintSetting = new ArrayList<>();
@@ -289,7 +302,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
//Fetch the service Components to obtain ServiceComponentInfo
Collection<TreeNode<Resource>> componentChildren = serviceNode.getChild("components").getChildren();
- for (TreeNode componentNode : componentChildren) {
+ for (TreeNode<Resource> componentNode : componentChildren) {
ResourceImpl component = (ResourceImpl) componentNode.getObject();
Map<String, Object> serviceComponentInfoMap = component.getPropertiesMap().get("ServiceComponentInfo");
String componentName = serviceComponentInfoMap.get("component_name").toString();
@@ -307,8 +320,13 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
}
}
}
- blueprintSetting.add(ImmutableMap.of(SERVICE_SETTINGS, serviceSettings));
- blueprintSetting.add(ImmutableMap.of(COMPONENT_SETTINGS, componentSettings));
+
+ if (exportType.include(serviceSettings)) {
+ blueprintSetting.add(ImmutableMap.of(SERVICE_SETTINGS, serviceSettings));
+ }
+ if (exportType.include(componentSettings)) {
+ blueprintSetting.add(ImmutableMap.of(COMPONENT_SETTINGS, componentSettings));
+ }
return blueprintSetting;
}
@@ -366,14 +384,23 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
List<Map<String, Map<String, Map<String, ?>>>> configList = new ArrayList<>();
Configuration configuration = topology.getConfiguration();
- Collection<String> allTypes = new HashSet<>();
- allTypes.addAll(configuration.getFullProperties().keySet());
- allTypes.addAll(configuration.getFullAttributes().keySet());
+ Map<String, Map<String, String>> fullProperties = configuration.getFullProperties();
+ Map<String, Map<String, Map<String, String>>> fullAttributes = configuration.getFullAttributes();
+ Collection<String> allTypes = ImmutableSet.<String>builder()
+ .addAll(fullProperties.keySet())
+ .addAll(fullAttributes.keySet())
+ .build();
for (String type : allTypes) {
Map<String, Map<String, ?>> typeMap = new HashMap<>();
- typeMap.put("properties", configuration.getFullProperties().get(type));
- if (! configuration.getFullAttributes().isEmpty()) {
- typeMap.put("properties_attributes", configuration.getFullAttributes().get(type));
+ Map<String, String> properties = fullProperties.get(type);
+ if (exportType.include(properties)) {
+ typeMap.put("properties", properties);
+ }
+ if (!fullAttributes.isEmpty()) {
+ Map<String, Map<String, String>> attributes = fullAttributes.get(type);
+ if (exportType.include(attributes)) {
+ typeMap.put("properties_attributes", attributes);
+ }
}
configList.add(Collections.singletonMap(type, typeMap));
@@ -407,7 +434,9 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
configList.add(propertyMap);
}
- mapGroupProperties.put("configurations", configList);
+ if (exportType.include(configList)) {
+ mapGroupProperties.put("configurations", configList);
+ }
}
return listHostGroups;
}
@@ -433,7 +462,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
protected ClusterTopology createClusterTopology(TreeNode<Resource> clusterNode)
throws InvalidTopologyTemplateException, InvalidTopologyException {
- return new ClusterTopologyImpl(new AmbariContext(), new ExportBlueprintRequest(clusterNode));
+ return new ClusterTopologyImpl(new AmbariContext(), new ExportBlueprintRequest(clusterNode, controller));
}
/**
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
index bc0e3ae..cf9426c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
@@ -20,11 +20,13 @@ package org.apache.ambari.server.api.resources;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Optional;
import java.util.Set;
import org.apache.ambari.server.api.query.render.ClusterBlueprintRenderer;
import org.apache.ambari.server.api.query.render.Renderer;
import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.internal.BlueprintExportType;
import org.apache.ambari.server.controller.spi.Resource;
/**
@@ -51,11 +53,10 @@ public class ClusterResourceDefinition extends BaseResourceDefinition {
@Override
public Renderer getRenderer(String name) {
- if (name != null && name.equals("blueprint")) {
- return new ClusterBlueprintRenderer();
- } else {
- return super.getRenderer(name);
- }
+ Optional<BlueprintExportType> blueprintExportType = BlueprintExportType.parse(name);
+ return blueprintExportType.isPresent()
+ ? new ClusterBlueprintRenderer(blueprintExportType.get())
+ : super.getRenderer(name);
}
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 2d580e4..c15ca70 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -63,8 +63,10 @@ import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -196,12 +198,22 @@ public class BlueprintConfigurationProcessor {
private static final Set<String> configPropertiesWithHASupport =
new HashSet<>(Arrays.asList("fs.defaultFS", "hbase.rootdir", "instance.volumes", "policymgr_external_url", "xasecure.audit.destination.hdfs.dir"));
+ private static final Set<Pair<String, String>> PROPERTIES_FOR_HADOOP_PROXYUSER = ImmutableSet.of(
+ Pair.of("oozie-env", "oozie_user"),
+ Pair.of("hive-env", "hive_user"),
+ Pair.of("hive-env", "webhcat_user"),
+ Pair.of("hbase-env", "hbase_user"),
+ Pair.of("falcon-env", "falcon_user")
+ );
+ private static final String HADOOP_PROXYUSER_HOSTS_FORMAT = "hadoop.proxyuser.%s.hosts";
+ private static final String HADOOP_PROXYUSER_GROUPS_FORMAT = "hadoop.proxyuser.%s.groups";
+
/**
* Statically-defined list of filters to apply on property exports.
* This will initially be used to filter out the Ranger Passwords, but
* could be extended in the future for more generic purposes.
*/
- private PropertyFilter[] getExportPropertyFilters (Map<Long, Set<String>> authToLocalPerClusterMap)
+ private PropertyFilter[] getExportPropertyFilters(Map<Long, Set<String>> authToLocalPerClusterMap)
{
return new PropertyFilter[] {
new PasswordPropertyFilter(),
@@ -451,9 +463,8 @@ public class BlueprintConfigurationProcessor {
}
// Explicitly set any properties that are required but not currently provided in the stack definition.
+ injectDefaults(clusterConfig, configTypesUpdated, clusterTopology.getBlueprint().getServices());
setStackToolsAndFeatures(clusterConfig, configTypesUpdated);
- setRetryConfiguration(clusterConfig, configTypesUpdated);
- setupHDFSProxyUsers(clusterConfig, configTypesUpdated);
addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getBlueprint().getStack());
trimProperties(clusterConfig, clusterTopology);
@@ -815,7 +826,7 @@ public class BlueprintConfigurationProcessor {
* Update properties for blueprint export.
* This involves converting concrete topology information to host groups.
*/
- public void doUpdateForBlueprintExport() {
+ public void doUpdateForBlueprintExport(BlueprintExportType exportType) {
// HA configs are only processed in cluster configuration, not HG configurations
if (clusterTopology.isNameNodeHAEnabled()) {
doNameNodeHAUpdate();
@@ -830,7 +841,8 @@ public class BlueprintConfigurationProcessor {
}
Collection<Configuration> allConfigs = new ArrayList<>();
- allConfigs.add(clusterTopology.getConfiguration());
+ Configuration clusterConfig = clusterTopology.getConfiguration();
+ allConfigs.add(clusterConfig);
for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
Configuration hgConfiguration = groupInfo.getConfiguration();
if (! hgConfiguration.getFullProperties(1).isEmpty()) {
@@ -852,6 +864,20 @@ public class BlueprintConfigurationProcessor {
doFilterPriorToExport(configuration);
}
+
+ Blueprint blueprint = clusterTopology.getBlueprint();
+ applyTypeSpecificFilter(exportType, clusterConfig, blueprint.getStack().getConfiguration(), blueprint.getServices());
+ }
+
+ @VisibleForTesting
+ void applyTypeSpecificFilter(BlueprintExportType exportType, Configuration clusterConfig, Configuration stackConfig, Collection<String> services) {
+ if (exportType == BlueprintExportType.MINIMAL) {
+ // convert back to suffix-less form, to allow comparing to defaults
+ doNonTopologyUpdate(mPropertyUpdaters, clusterConfig);
+ }
+
+ injectDefaults(stackConfig, new HashSet<>(), services);
+ exportType.filter(clusterConfig, stackConfig);
}
/**
@@ -1673,11 +1699,11 @@ public class BlueprintConfigurationProcessor {
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
for (String propertyName : entry.getValue().keySet()) {
- NonTopologyUpdater npu = (NonTopologyUpdater) entry.getValue().get(propertyName);
+ PropertyUpdater pu = entry.getValue().get(propertyName);
Map<String, String> typeProperties = properties.get(type);
if (typeProperties != null && typeProperties.containsKey(propertyName)) {
- String newValue = npu.updateForBlueprintExport(propertyName, typeProperties.get(propertyName), properties, clusterTopology);
+ String newValue = pu.updateForBlueprintExport(propertyName, typeProperties.get(propertyName), properties, clusterTopology);
configuration.setProperty(type, propertyName, newValue);
}
}
@@ -1703,6 +1729,10 @@ public class BlueprintConfigurationProcessor {
Map<String, Map<String, String>> properties,
ClusterTopology topology);
+ default String updateForBlueprintExport(String propertyName, String value, Map<String, Map<String, String>> properties, ClusterTopology topology) {
+ return value;
+ }
+
/**
* Determine the required host groups for the provided property.
*
@@ -2764,13 +2794,6 @@ public class BlueprintConfigurationProcessor {
ClusterTopology topology) {
return Collections.emptyList();
}
-
- public String updateForBlueprintExport(String propertyName,
- String origValue,
- Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
- return origValue;
- }
}
@@ -3228,52 +3251,75 @@ public class BlueprintConfigurationProcessor {
}
}
- private Collection<String> setupHDFSProxyUsers(Configuration configuration, Set<String> configTypesUpdated) {
- // AMBARI-5206
- final Map<String , String> userProps = new HashMap<>();
+ /**
+ * Generates property names of the format "hadoop.proxyuser.*" based on actual usernames defined in {@code configuration}.
+ * Eg. if "hive-env" defined "hive_user": "cstm-hive", then a generated property name would be "hadoop_proxyuser_cstm-hive_hosts"
+ * @return set of hadoop proxyuser property names, paired with the name of the config type the username is defined in
+ * (the config type is needed for filtering later)
+ */
+ private static Set<Pair<String, String>> generateHadoopProxyUserPropertyNames(Configuration configuration) {
+ Set<Pair<String, String>> proxyUsers = new HashSet<>();
+ Map<String, Map<String, String>> existingProperties = configuration.getFullProperties();
+ for (Pair<String, String> userProp : PROPERTIES_FOR_HADOOP_PROXYUSER) {
+ String configType = userProp.getLeft();
+ String property = userProp.getRight();
+ Map<String, String> configs = existingProperties.get(configType);
+ if (configs != null) {
+ String user = configs.get(property);
+ if (!Strings.isNullOrEmpty(user)) {
+ proxyUsers.add(Pair.of(configType, String.format(HADOOP_PROXYUSER_HOSTS_FORMAT, user)));
+ proxyUsers.add(Pair.of(configType, String.format(HADOOP_PROXYUSER_GROUPS_FORMAT, user)));
+ }
+ }
+ }
+
+ return proxyUsers;
+ }
- Collection<String> services = clusterTopology.getBlueprint().getServices();
+ /**
+ * Ensures {@code hadoop.proxyuser.*} properties are present in core-site for the services defined in the blueprint.
+ */
+ private static void setupHDFSProxyUsers(Configuration configuration, Set<String> configTypesUpdated, Collection<String> services) {
if (services.contains("HDFS")) {
- // only add user properties to the map for
- // services actually included in the blueprint definition
- if (services.contains("OOZIE")) {
- userProps.put("oozie_user", "oozie-env");
- }
+ Set<Pair<String, String>> configTypePropertyPairs = generateHadoopProxyUserPropertyNames(configuration);
+ Set<String> acceptableConfigTypes = getEligibleConfigTypesForHadoopProxyUsers(services);
- if (services.contains("HIVE")) {
- userProps.put("hive_user", "hive-env");
- userProps.put("webhcat_user", "hive-env");
+ Map<String, Map<String, String>> existingProperties = configuration.getFullProperties();
+ for (Pair<String, String> pair : configTypePropertyPairs) {
+ String configType = pair.getLeft();
+ if (acceptableConfigTypes.contains(configType)) {
+ Map<String, String> configs = existingProperties.get(configType);
+ if (configs != null) {
+ ensureProperty(configuration, "core-site", pair.getRight(), "*", configTypesUpdated);
+ }
+ }
}
+ }
+ }
- if (services.contains("HBASE")) {
- userProps.put("hbase_user", "hbase-env");
- }
+ /**
+ * @return set of config types with eligible properties for hadoop proxyuser
+ */
+ private static Set<String> getEligibleConfigTypesForHadoopProxyUsers(Collection<String> services) {
+ Set<String> acceptableConfigTypes = new HashSet<>();
- if (services.contains("FALCON")) {
- userProps.put("falcon_user", "falcon-env");
- }
+ if (services.contains("OOZIE")) {
+ acceptableConfigTypes.add("oozie-env");
+ }
- String proxyUserHosts = "hadoop.proxyuser.%s.hosts";
- String proxyUserGroups = "hadoop.proxyuser.%s.groups";
+ if (services.contains("HIVE")) {
+ acceptableConfigTypes.add("hive-env");
+ }
- Map<String, Map<String, String>> existingProperties = configuration.getFullProperties();
- for (String property : userProps.keySet()) {
- String configType = userProps.get(property);
- Map<String, String> configs = existingProperties.get(configType);
- if (configs != null) {
- String user = configs.get(property);
- if (user != null && !user.isEmpty()) {
- ensureProperty(configuration, "core-site", String.format(proxyUserHosts, user), "*", configTypesUpdated);
- ensureProperty(configuration, "core-site", String.format(proxyUserGroups, user), "*", configTypesUpdated);
- }
- } else {
- LOG.debug("setMissingConfigurations: no user configuration found for type = {}. This may be caused by an error in the blueprint configuration.",
- configType);
- }
+ if (services.contains("HBASE")) {
+ acceptableConfigTypes.add("hbase-env");
+ }
- }
+ if (services.contains("FALCON")) {
+ acceptableConfigTypes.add("falcon-env");
}
- return services;
+
+ return acceptableConfigTypes;
}
/**
@@ -3416,6 +3462,14 @@ public class BlueprintConfigurationProcessor {
}
}
+ /**
+ * Ensures that properties non-stack properties are present in {@code configuration}.
+ */
+ private static void injectDefaults(Configuration configuration, Set<String> configTypesUpdated, Collection<String> services) {
+ setRetryConfiguration(configuration, configTypesUpdated);
+ setupHDFSProxyUsers(configuration, configTypesUpdated, services);
+ }
+
private @Nullable String trimValue(@Nullable String value,
@NotNull Stack stack,
@NotNull String configType,
@@ -3439,7 +3493,7 @@ public class BlueprintConfigurationProcessor {
* @param property property name
* @param defaultValue default value
*/
- private void ensureProperty(Configuration configuration, String type, String property, String defaultValue, Set<String> configTypesUpdated) {
+ private static void ensureProperty(Configuration configuration, String type, String property, String defaultValue, Set<String> configTypesUpdated) {
if (configuration.getPropertyValue(type, property) == null) {
configuration.setProperty(type, property, defaultValue);
configTypesUpdated.add(type);
@@ -3503,6 +3557,7 @@ public class BlueprintConfigurationProcessor {
return !PASSWORD_NAME_REGEX.matcher(propertyName).matches();
}
}
+
/**
* A Filter that excludes properties if in stack a property is marked as password property or kerberos principal
*
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintExportType.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintExportType.java
new file mode 100644
index 0000000..5bada66
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintExportType.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.internal;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Handles most of type-specific behavior for blueprint export.
+ */
+public enum BlueprintExportType {
+ /**
+ * The exported blueprint contains all properties of all config types
+ * for services present in the cluster.
+ */
+ FULL {
+ @Override
+ public Configuration filter(Configuration actualConfig, Configuration defaultConfig) {
+ // no-op
+ return actualConfig;
+ }
+
+ @Override
+ public boolean include(String value, String defaultValue) {
+ return true;
+ }
+
+ @Override
+ public boolean include(Collection<?> collection) {
+ return true;
+ }
+
+ @Override
+ public boolean include(Map<?, ?> map) {
+ return true;
+ }
+ },
+
+ /**
+ * The exported blueprint contains only the properties that do not match default values
+ * as defined in the stack. Empty lists/maps are also omitted.
+ */
+ MINIMAL {
+ @Override
+ public Configuration filter(Configuration actualConfig, Configuration defaultConfig) {
+ for (Map.Entry<String, Map<String, String>> configTypeEntry : ImmutableSet.copyOf(actualConfig.getProperties().entrySet())) {
+ String configType = configTypeEntry.getKey();
+ Map<String, String> properties = configTypeEntry.getValue();
+ for (Map.Entry<String, String> propertyEntry : ImmutableSet.copyOf(properties.entrySet())) {
+ String propertyName = propertyEntry.getKey();
+ String propertyValue = propertyEntry.getValue();
+ String defaultValue = defaultConfig.getPropertyValue(configType, propertyName);
+ if (include(propertyValue, defaultValue)) {
+ LOG.debug("Including {}/{} in exported blueprint, as default value and actual value differ:\n{}\nvs\n{}", configType, propertyName, defaultValue, propertyValue);
+ } else {
+ LOG.debug("Omitting {}/{} from exported blueprint, as it has the default value of {}", configType, propertyName, propertyValue);
+ actualConfig.removeProperty(configType, propertyName);
+ }
+ }
+ if (properties.isEmpty()) {
+ actualConfig.getProperties().remove(configType);
+ }
+ }
+
+ for (Map.Entry<String, Map<String, Map<String, String>>> configTypeEntry : ImmutableSet.copyOf(actualConfig.getAttributes().entrySet())) {
+ String configType = configTypeEntry.getKey();
+ Map<String, Map<String, String>> attributes = configTypeEntry.getValue();
+ for (Map.Entry<String, Map<String, String>> attributeEntry : ImmutableSet.copyOf(attributes.entrySet())) {
+ String attributeName = attributeEntry.getKey();
+ Map<String, String> properties = attributeEntry.getValue();
+ for (Map.Entry<String, String> propertyEntry : ImmutableSet.copyOf(properties.entrySet())) {
+ String propertyName = propertyEntry.getKey();
+ String attributeValue = propertyEntry.getValue();
+ String defaultValue = defaultConfig.getAttributeValue(configType, propertyName, attributeName);
+ if (include(attributeValue, defaultValue)) {
+ LOG.debug("Including {}/{}/{} in exported blueprint, as default value and actual value differ:\n{}\nvs\n{}", configType, attributeName, propertyName, defaultValue, attributeValue);
+ } else {
+ LOG.debug("Omitting {}/{}/{} from exported blueprint, as it has the default value of {}", configType, attributeName, propertyName, attributeValue);
+ properties.remove(propertyName);
+ }
+ }
+ if (properties.isEmpty()) {
+ attributes.remove(attributeName);
+ }
+ }
+ if (attributes.isEmpty()) {
+ actualConfig.getAttributes().remove(configType);
+ }
+ }
+
+ return actualConfig;
+ }
+
+ @Override
+ public boolean include(String value, String defaultValue) {
+ return value != null && (
+ defaultValue == null ||
+ !Objects.equals(StringUtils.trim(defaultValue), StringUtils.trim(value))
+ );
+ }
+
+ @Override
+ public boolean include(Collection<?> collection) {
+ return collection != null && !collection.isEmpty();
+ }
+
+ @Override
+ public boolean include(Map<?, ?> map) {
+ return map != null && !map.isEmpty();
+ }
+ },
+ ;
+
+ public abstract Configuration filter(Configuration actualConfig, Configuration defaultConfig);
+ public abstract boolean include(String value, String defaultValue);
+ public abstract boolean include(Collection<?> collection);
+ public abstract boolean include(Map<?, ?> map);
+
+ public static final BlueprintExportType DEFAULT = MINIMAL;
+ public static final String PREFIX = "blueprint";
+ private static final String SEPARATOR = "_";
+ private static final Logger LOG = LoggerFactory.getLogger(BlueprintExportType.class);
+
+ public static Optional<BlueprintExportType> parse(String input) {
+ if (input == null || !input.startsWith(PREFIX)) {
+ return Optional.empty();
+ }
+
+ int separatorPos = input.indexOf(SEPARATOR);
+ if (separatorPos == -1 || separatorPos + 1 == input.length()) {
+ return Optional.of(DEFAULT);
+ }
+
+ switch (input.substring(separatorPos + 1)) {
+ case "full": return Optional.of(FULL);
+ case "minimal": return Optional.of(MINIMAL);
+ default: return Optional.of(DEFAULT);
+ }
+ }
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
index e604bf3..bf010de 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
@@ -32,7 +32,6 @@ import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.util.TreeNode;
import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.state.DesiredConfig;
@@ -52,31 +51,31 @@ import org.apache.ambari.server.topology.TopologyRequest;
*/
public class ExportBlueprintRequest implements TopologyRequest {
- private static AmbariManagementController controller = AmbariServer.getController();
+ private final AmbariManagementController controller;
- private String clusterName;
- private Long clusterId;
+ private final String clusterName;
+ private final Long clusterId;
private Blueprint blueprint;
- private Configuration configuration;
- //todo: Should this map be represented by a new class?
- private Map<String, HostGroupInfo> hostGroupInfo = new HashMap<>();
+ private final Configuration configuration;
+ private final Map<String, HostGroupInfo> hostGroupInfo = new HashMap<>();
- public ExportBlueprintRequest(TreeNode<Resource> clusterNode) throws InvalidTopologyTemplateException {
+ public ExportBlueprintRequest(TreeNode<Resource> clusterNode, AmbariManagementController controller) throws InvalidTopologyTemplateException {
+ this.controller = controller;
+
Resource clusterResource = clusterNode.getObject();
+ Stack stack = parseStack(clusterResource);
clusterName = String.valueOf(clusterResource.getPropertyValue(
ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID));
clusterId = Long.valueOf(String.valueOf(clusterResource.getPropertyValue(
ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID)));
- createConfiguration(clusterNode);
- //todo: should be parsing Configuration from the beginning
- //createConfiguration(configurations);
+ configuration = createConfiguration(clusterNode);
Collection<ExportedHostGroup> exportedHostGroups = processHostGroups(clusterNode.getChild("hosts"));
createHostGroupInfo(exportedHostGroups);
- createBlueprint(exportedHostGroups, parseStack(clusterResource));
+ createBlueprint(exportedHostGroups, stack);
}
public String getClusterName() {
@@ -138,7 +137,6 @@ public class ExportBlueprintRequest implements TopologyRequest {
for (ExportedHostGroup exportedGroup : exportedHostGroups) {
HostGroupInfo groupInfo = new HostGroupInfo(exportedGroup.getName());
groupInfo.addHosts(exportedGroup.getHostInfo());
- //todo: should be parsing Configuration from the beginning
groupInfo.setConfiguration(exportedGroup.getConfiguration());
hostGroupInfo.put(groupInfo.getHostGroupName(), groupInfo);
}
@@ -161,10 +159,8 @@ public class ExportBlueprintRequest implements TopologyRequest {
* Process cluster scoped configurations.
*
* @param clusterNode cluster node
- *
*/
- private void createConfiguration(TreeNode<Resource> clusterNode) {
-
+ private static Configuration createConfiguration(TreeNode<Resource> clusterNode) {
Map<String, Map<String, String>> properties = new HashMap<>();
Map<String, Map<String, Map<String, String>>> attributes = new HashMap<>();
@@ -174,16 +170,15 @@ public class ExportBlueprintRequest implements TopologyRequest {
ExportedConfiguration configuration = new ExportedConfiguration(config);
DesiredConfig desiredConfig = (DesiredConfig) desiredConfigMap.get(configuration.getType());
if (desiredConfig != null && desiredConfig.getTag().equals(configuration.getTag())) {
-
properties.put(configuration.getType(), configuration.getProperties());
attributes.put(configuration.getType(), configuration.getPropertyAttributes());
}
}
- configuration = new Configuration(properties, attributes);
- // empty parent configuration when exporting as all properties are included in this configuration
- configuration.setParentConfiguration(new Configuration(
- Collections.emptyMap(),
- Collections.emptyMap()));
+
+ Configuration configuration = new Configuration(properties, attributes);
+ configuration.setParentConfiguration(new Configuration(Collections.emptyMap(), Collections.emptyMap()));
+
+ return configuration;
}
/**
@@ -231,12 +226,12 @@ public class ExportBlueprintRequest implements TopologyRequest {
/**
* Associated components.
*/
- private Set<String> components = new HashSet<>();
+ private final Set<String> components = new HashSet<>();
/**
* Host group scoped configurations.
*/
- private Collection<ExportedConfiguration> configurations = new HashSet<>();
+ private final Collection<ExportedConfiguration> configurations = new HashSet<>();
/**
* Number of instances.
@@ -246,7 +241,7 @@ public class ExportBlueprintRequest implements TopologyRequest {
/**
* Collection of associated hosts.
*/
- private Collection<String> hosts = new HashSet<>();
+ private final Collection<String> hosts = new HashSet<>();
/**
* Constructor.
@@ -376,7 +371,6 @@ public class ExportBlueprintRequest implements TopologyRequest {
getComponents().add("AMBARI_SERVER");
}
} catch (UnknownHostException e) {
- //todo: SystemException?
throw new RuntimeException("Unable to obtain local host name", e);
}
} catch (UnknownHostException e) {
@@ -406,7 +400,7 @@ public class ExportBlueprintRequest implements TopologyRequest {
/**
* Encapsulates a configuration.
*/
- private class ExportedConfiguration {
+ private static class ExportedConfiguration {
/**
* Configuration type such as hdfs-site.
*/
@@ -448,18 +442,6 @@ public class ExportBlueprintRequest implements TopologyRequest {
if (propertiesMap.containsKey("properties_attributes")) {
propertyAttributes = (Map) propertiesMap.get("properties_attributes");
}
-
- //todo: not processing config here, ensure that
- //todo: this logic regarding null/empty properties is properly handled
-// if (properties != null && !properties.isEmpty()) {
-// stripRequiredProperties(properties);
-// } else {
-// LOG.warn("Empty configuration found for configuration type = " + type +
-// " during Blueprint export. This may occur after an upgrade of Ambari, when" +
-// "attempting to export a Blueprint from a cluster started by an older version of " +
-// "Ambari.");
-// }
-
}
/**
@@ -517,4 +499,5 @@ public class ExportBlueprintRequest implements TopologyRequest {
return result;
}
}
+
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
index 8b7cb67..501e16a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
@@ -55,11 +55,21 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda
return value.toString();
} else if (!value.hasAnyUnit()) {
return value.withUnit(stackUnit);
- } else { // should not happen because of prevalidation in UnitValidator
+ } else { // should not happen because of pre-validation in UnitValidator
throw new IllegalArgumentException("Property " + propertyName + "=" + origValue + " has an unsupported unit. Stack supported unit is: " + stackUnit + " or no unit");
}
}
+ /**
+ * @return property value with removed unit
+ */
+ @Override
+ public String updateForBlueprintExport(String propertyName, String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) {
+ PropertyUnit stackUnit = PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType, propertyName);
+ PropertyValue value = PropertyValue.of(propertyName, origValue);
+ return value.withoutUnit(stackUnit);
+ }
+
@Override
public Collection<String> getRequiredHostGroups(String propertyName, String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) {
return Collections.emptySet();
@@ -142,6 +152,12 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda
return value + unit;
}
+ public String withoutUnit(PropertyUnit unit) {
+ return hasUnit(unit)
+ ? value.substring(0, value.length() - unit.toString().length())
+ : value;
+ }
+
@Override
public String toString() {
return value;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
index 31fcb9d..a3564fb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
@@ -187,6 +187,13 @@ public class PropertyInfo {
for (Element propertyAttribute : propertyAttributes) {
attributes.put(propertyAttribute.getTagName(), propertyAttribute.getFirstChild().getNodeValue());
}
+
+ // inject "hidden" property_value_attribute into property_attributes, see AMBARI-17223
+ String hidden = getPropertyValueAttributes().getHidden();
+ if (hidden != null) {
+ attributes.putIfAbsent("hidden", hidden);
+ }
+
return attributes;
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 4026ce6..bbd4502 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -43,6 +43,18 @@ public class Configuration {
*/
private Configuration parentConfiguration;
+ public static Configuration newEmpty() {
+ return new Configuration(new HashMap<>(), new HashMap<>());
+ }
+
+ public Configuration copy() {
+ Configuration parent = parentConfiguration;
+ parentConfiguration = null;
+ Configuration copy = new Configuration(getFullProperties(), getFullAttributes());
+ parentConfiguration = parent;
+ return copy;
+ }
+
/**
* Constructor.
*
@@ -385,10 +397,10 @@ public class Configuration {
* Remove all occurrences of a config type
*/
public void removeConfigType(String configType) {
- if (properties != null && properties.containsKey(configType)) {
+ if (properties != null) {
properties.remove(configType);
}
- if (attributes != null && attributes.containsKey(configType)) {
+ if (attributes != null) {
attributes.remove(configType);
}
if (parentConfiguration != null) {
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
index b9ef9c9..7ccbfc3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
@@ -59,6 +59,7 @@ import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.KerberosHelperImpl;
import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
+import org.apache.ambari.server.controller.internal.BlueprintExportType;
import org.apache.ambari.server.controller.internal.ClusterControllerImpl;
import org.apache.ambari.server.controller.internal.ResourceImpl;
import org.apache.ambari.server.controller.internal.Stack;
@@ -124,6 +125,9 @@ public class ClusterBlueprintRendererTest {
new HashMap<>();
private static final Configuration clusterConfig = new Configuration(clusterProps, clusterAttributes);
+ private final ClusterBlueprintRenderer minimalRenderer = new ClusterBlueprintRenderer(BlueprintExportType.MINIMAL);
+ private final ClusterBlueprintRenderer fullRenderer = new ClusterBlueprintRenderer(BlueprintExportType.FULL);
+
@Before
public void setup() throws Exception {
@@ -164,8 +168,10 @@ public class ClusterBlueprintRendererTest {
expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes();
expect(blueprint.getHostGroup("host_group_1")).andReturn(group1).anyTimes();
expect(blueprint.getHostGroup("host_group_2")).andReturn(group2).anyTimes();
+ expect(blueprint.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN")).anyTimes();
expect(stack.getName()).andReturn("HDP").anyTimes();
expect(stack.getVersion()).andReturn("1.3.3").anyTimes();
+ expect(stack.getConfiguration()).andReturn(Configuration.newEmpty()).anyTimes();
expect(group1.getName()).andReturn("host_group_1").anyTimes();
expect(group2.getName()).andReturn("host_group_2").anyTimes();
expect(group1.getComponents()).andReturn(group1Components).anyTimes();
@@ -256,8 +262,7 @@ public class ClusterBlueprintRendererTest {
QueryInfo hostComponentInfo = new QueryInfo(new HostComponentResourceDefinition(), new HashSet<>());
queryTree.getChild("Host").addChild(hostComponentInfo, "HostComponent");
- ClusterBlueprintRenderer renderer = new ClusterBlueprintRenderer();
- TreeNode<Set<String>> propertyTree = renderer.finalizeProperties(queryTree, false);
+ TreeNode<Set<String>> propertyTree = fullRenderer.finalizeProperties(queryTree, false);
Set<String> rootProperties = propertyTree.getObject();
assertEquals(2, rootProperties.size());
@@ -276,12 +281,13 @@ public class ClusterBlueprintRendererTest {
defaultCredentialStoreSettings(),
defaultRecoverySettings());
- Collection<Map<String, Object>> settings = ClusterBlueprintRenderer.getSettings(clusterNode, stack);
-
+ Collection<Map<String, Object>> settings = fullRenderer.getSettings(clusterNode, stack);
assertEquals(Lists.newArrayList(
ImmutableMap.of(ClusterBlueprintRenderer.SERVICE_SETTINGS, ImmutableSet.of()),
ImmutableMap.of(ClusterBlueprintRenderer.COMPONENT_SETTINGS, ImmutableSet.of())
), settings);
+
+ assertEquals(ImmutableList.of(), minimalRenderer.getSettings(clusterNode, stack));
}
@Test
@@ -292,8 +298,7 @@ public class ClusterBlueprintRendererTest {
customCredentialStoreSettingFor(stack, "service1", "service2"),
customRecoverySettingsFor(stack, "component1", "component2"));
- Collection<Map<String, Object>> settings = ClusterBlueprintRenderer.getSettings(clusterNode, stack);
-
+ Collection<Map<String, Object>> settings = fullRenderer.getSettings(clusterNode, stack);
assertEquals(Lists.newArrayList(
ImmutableMap.of(ClusterBlueprintRenderer.SERVICE_SETTINGS, ImmutableSet.of(
ImmutableMap.of(
@@ -310,6 +315,8 @@ public class ClusterBlueprintRendererTest {
ImmutableMap.of("name", "component2", ClusterBlueprintRenderer.RECOVERY_ENABLED, ClusterBlueprintRenderer.TRUE)
))
), settings);
+
+ assertEquals(settings, minimalRenderer.getSettings(clusterNode, stack));
}
@Test
@@ -320,7 +327,7 @@ public class ClusterBlueprintRendererTest {
defaultCredentialStoreSettings(),
customRecoverySettingsFor(stack, "component1"));
- Collection<Map<String, Object>> settings = ClusterBlueprintRenderer.getSettings(clusterNode, stack);
+ Collection<Map<String, Object>> settings = fullRenderer.getSettings(clusterNode, stack);
assertEquals(Lists.newArrayList(
ImmutableMap.of(ClusterBlueprintRenderer.SERVICE_SETTINGS, ImmutableSet.of()),
@@ -328,6 +335,11 @@ public class ClusterBlueprintRendererTest {
ImmutableMap.of("name", "component1", ClusterBlueprintRenderer.RECOVERY_ENABLED, ClusterBlueprintRenderer.FALSE)
))
), settings);
+ assertEquals(Lists.newArrayList(
+ ImmutableMap.of(ClusterBlueprintRenderer.COMPONENT_SETTINGS, ImmutableSet.of(
+ ImmutableMap.of("name", "component1", ClusterBlueprintRenderer.RECOVERY_ENABLED, ClusterBlueprintRenderer.FALSE)
+ ))
+ ), minimalRenderer.getSettings(clusterNode, stack));
}
private static TreeNode<Resource> clusterWith(Stack stack,
@@ -456,8 +468,7 @@ public class ClusterBlueprintRendererTest {
rootQuery.getProperties().add("foo/bar");
rootQuery.getProperties().add("prop1");
- ClusterBlueprintRenderer renderer = new ClusterBlueprintRenderer();
- TreeNode<Set<String>> propertyTree = renderer.finalizeProperties(queryTree, false);
+ TreeNode<Set<String>> propertyTree = fullRenderer.finalizeProperties(queryTree, false);
Set<String> rootProperties = propertyTree.getObject();
assertEquals(2, rootProperties.size());
@@ -476,7 +487,7 @@ public class ClusterBlueprintRendererTest {
Result result = new ResultImpl(true);
createClusterResultTree(result.getResultTree());
- ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology);
+ ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology, BlueprintExportType.FULL);
Result blueprintResult = renderer.finalizeResult(result);
TreeNode<Resource> blueprintTree = blueprintResult.getResultTree();
@@ -502,7 +513,7 @@ public class ClusterBlueprintRendererTest {
Result result = new ResultImpl(true);
createClusterResultTree(result.getResultTree());
- ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology);
+ ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology, BlueprintExportType.FULL);
Result blueprintResult = renderer.finalizeResult(result);
TreeNode<Resource> blueprintTree = blueprintResult.getResultTree();
@@ -575,7 +586,7 @@ public class ClusterBlueprintRendererTest {
createClusterResultTree(result.getResultTree(), testDesiredConfigMap);
- ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology);
+ ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology, BlueprintExportType.FULL);
Result blueprintResult = renderer.finalizeResult(result);
TreeNode<Resource> blueprintTree = blueprintResult.getResultTree();
@@ -678,11 +689,8 @@ public class ClusterBlueprintRendererTest {
@Test
public void testClusterRendererDefaults() {
- Renderer clusterBlueprintRenderer =
- new ClusterBlueprintRenderer();
-
assertFalse("ClusterBlueprintRenderer should not require property provider input",
- clusterBlueprintRenderer.requiresPropertyProviderInput());
+ fullRenderer.requiresPropertyProviderInput());
}
//todo: collection resource
@@ -827,7 +835,8 @@ public class ClusterBlueprintRendererTest {
private ClusterTopology topology;
- TestBlueprintRenderer(ClusterTopology topology) {
+ TestBlueprintRenderer(ClusterTopology topology, BlueprintExportType exportType) {
+ super(exportType);
this.topology = topology;
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 996c09d..28a9bcd 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -167,6 +167,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// return false for all components since for this test we don't care about the value
expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes();
expect(stack.getConfigurationPropertiesWithMetadata(anyObject(String.class), anyObject(String.class))).andReturn(emptyMap()).anyTimes();
+ expect(stack.getConfiguration()).andReturn(Configuration.newEmpty()).anyTimes();
expect(serviceInfo.getRequiredProperties()).andReturn(
emptyMap()).anyTimes();
@@ -314,7 +315,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("yarn-site").get("yarn.resourcemanager.hostname");
assertEquals("%HOSTGROUP::group1%", updatedVal);
@@ -362,7 +363,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals(properties.size(), 3);
assertEquals(((Map) properties.get("kerberos-env")).size(), 0);
@@ -398,7 +399,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// When
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// Then
assertEquals("policymgr_external_url property's original value should be exported when Ranger Admin is deployed to multiple hosts.", "test_policymgr_external_url", properties.get("admin-properties").get("policymgr_external_url"));
@@ -440,7 +441,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("%HOSTGROUP::group1%", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname"));
assertEquals("%HOSTGROUP::group1%", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address"));
@@ -488,7 +489,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("%HOSTGROUP::group1%", properties.get("yarn-site").get("yarn.resourcemanager.hostname"));
assertEquals("%HOSTGROUP::group1%",
@@ -523,7 +524,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("core-site").get("fs.defaultFS");
assertEquals("%HOSTGROUP::group1%:8020", updatedVal);
@@ -556,7 +557,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertFalse(properties.get("yarn-site").containsKey("yarn.resourcemanager.hostname"));
}
@@ -602,7 +603,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("hbase-site").get("hbase.zookeeper.quorum");
assertEquals("%HOSTGROUP::group1%,%HOSTGROUP::group2%", updatedVal);
@@ -649,7 +650,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("webhcat-site").get("templeton.zookeeper.hosts");
assertEquals("%HOSTGROUP::group1%:5050,%HOSTGROUP::group2%:9090", updatedVal);
@@ -680,7 +681,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("application-properties").get("atlas.server.bind.address");
assertEquals("http://%HOSTGROUP::group1%:21000,http://%HOSTGROUP::group2%:21000", updatedVal);
@@ -735,7 +736,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("storm-site").get("storm.zookeeper.servers");
assertEquals("['%HOSTGROUP::group1%:5050','%HOSTGROUP::group2%:9090']", updatedVal);
@@ -778,7 +779,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("hive-site").get("javax.jdo.option.ConnectionURL");
assertEquals("jdbc:mysql://%HOSTGROUP::group1%/hive?createDatabaseIfNotExist=true", updatedVal);
@@ -811,7 +812,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertFalse(properties.get("hive-site").containsKey("javax.jdo.option.ConnectionURL"));
}
@@ -863,7 +864,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Exported properties map was not of the expected size", 2,
@@ -946,7 +947,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Falcon Broker URL property not properly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), falconStartupProperties.get("*.broker.url"));
@@ -983,7 +984,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertFalse("tez.tez-ui.history-url.base should not be present in exported blueprint in tez-site",
tezSiteProperties.containsKey("tez.tez-ui.history-url.base"));
@@ -1034,7 +1035,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// verify that these properties are filtered out of the exported configuration
assertFalse("admin_server_host should not be present in exported blueprint in kerberos-env",
@@ -1098,7 +1099,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("HTTPS address HA property not properly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne));
@@ -1171,7 +1172,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("HTTPS address HA property not properly exported",
createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.namenode.https-address." + expectedNameService + "." + expectedNodeOne));
@@ -1235,7 +1236,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// verify that any properties that include nameservices are not removed from the exported blueprint's configuration
assertEquals("Property containing an HA nameservice (fs.defaultFS), was not correctly exported by the processor",
@@ -1271,7 +1272,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Incorrect state for hdfs-site config after HA call in non-HA environment, should be zero",
0, hdfsSiteProperties.size());
@@ -1334,7 +1335,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// verify results for name service one
assertEquals("HTTPS address HA property not properly exported",
@@ -1413,7 +1414,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Yarn Log Server URL was incorrectly exported",
"http://" + "%HOSTGROUP::" + expectedHostGroupName + "%" + ":19888/jobhistory/logs", yarnSiteProperties.get("yarn.log.server.url"));
@@ -1485,7 +1486,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Yarn Log Server URL was incorrectly exported",
"http://" + "%HOSTGROUP::" + expectedHostGroupName + "%" + ":19888/jobhistory/logs", yarnSiteProperties.get("yarn.log.server.url"));
@@ -1565,7 +1566,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("hdfs config property not exported properly",
createExportedAddress(expectedPortNum, expectedHostGroupName), hdfsSiteProperties.get("dfs.http.address"));
@@ -1654,7 +1655,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("hive property not properly exported",
"thrift://" + createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("hive.metastore.uris"));
@@ -1747,7 +1748,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
System.out.println("RWN: exported value of hive.metastore.uris = " + hiveSiteProperties.get("hive.metastore.uris"));
@@ -1843,7 +1844,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// check that jdbc url and related properties are removed if oozie external db is on host which not included to cluster
assertFalse(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url"));
@@ -2008,7 +2009,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("zookeeper config not properly exported",
createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
@@ -2085,7 +2086,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Knox for core-site config not properly exported",
createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo),
@@ -2134,7 +2135,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("kafka Ganglia config not properly exported",
createExportedHostName(expectedHostGroupName, expectedPortNumberOne),
@@ -2173,7 +2174,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// call top-level export method
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("Property was incorrectly exported",
"%HOSTGROUP::" + expectedHostGroupName + "%", properties.get("storm.zookeeper.servers"));
@@ -3460,7 +3461,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
- updater.doUpdateForBlueprintExport();
+ updater.doUpdateForBlueprintExport(BlueprintExportType.FULL);
final String expectedPropertyValue = createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181", expectedHostGroupNameTwo);
assertEquals("hive.llap.zk.sm.connectionString property not updated correctly", expectedPropertyValue, hiveInteractiveSiteProperties.get(llapZkProperty));
@@ -3580,7 +3581,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
- updater.doUpdateForBlueprintExport();
+ updater.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertEquals("oozie property not updated correctly",
createExportedHostName(expectedHostGroupName, expectedPortNum), oozieSiteProperties.get("oozie.base.url"));
@@ -3731,7 +3732,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
- updater.doUpdateForBlueprintExport();
+ updater.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// verify that the properties with hostname information was correctly preserved
assertEquals("Yarn Log Server URL was incorrectly updated",
@@ -8135,7 +8136,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// When
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// Then
String expectedAuditHdfsDir = "hdfs://%HOSTGROUP::group2%:100";
@@ -8207,7 +8208,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
// When
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
// Then
String expectedAuditHdfsDir = "hdfs://my_name_service:100";
@@ -9044,7 +9045,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
properties.put("hive-site", typeProps);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String updatedVal = properties.get("hive-site").get("atlas.cluster.name");
assertEquals("primary", updatedVal);
@@ -9083,7 +9084,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
properties.put("storm-site", stormSiteProps);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
String hiveExecPostHooks = properties.get("hive-site").get("hive.exec.post.hooks");
String kafkaMetricsReporters = properties.get("kafka-broker").get("kafka.metrics.reporters");
@@ -9249,7 +9250,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(stack.isPasswordProperty((String) anyObject(), (String) anyObject(), (String) anyObject())).andReturn(true).once();
ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
- configProcessor.doUpdateForBlueprintExport();
+ configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL);
assertFalse(properties.get("ranger-admin-site").containsKey("ranger.service.https.attrib.keystore.pass"));
}
@@ -10039,6 +10040,66 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
assertNull(updatedRangerTagsyncSiteConfigurations);
}
+ @Test
+ public void defaultConfigs() {
+ Configuration stackConfig = createTestStack();
+ Configuration clusterConfig = stackConfig.copy();
+ Configuration customConfig = Configuration.newEmpty();
+
+ ClusterTopology topology = createNiceMock(ClusterTopology.class);
+ Stack stack = createNiceMock(Stack.class);
+ Collection<String> services = ImmutableList.of("HDFS");
+ expect(stack.getServices()).andReturn(services).anyTimes();
+ expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
+ expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes();
+ expect(topology.getHostGroupInfo()).andReturn(emptyMap()).anyTimes();
+ replay(stack, topology);
+
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+ updater.applyTypeSpecificFilter(BlueprintExportType.MINIMAL, clusterConfig, stackConfig, services);
+
+ assertEquals(customConfig.getProperties(), clusterConfig.getProperties());
+ }
+
+ @Test
+ public void customConfigs() {
+ Configuration stackConfig = createTestStack();
+ Configuration clusterConfig = stackConfig.copy();
+ Configuration customConfig = Configuration.newEmpty();
+ customize(clusterConfig, customConfig, "core-site", "hadoop.security.authorization", "true");
+ customize(clusterConfig, customConfig, "core-site", "fs.trash.interval", "0");
+ customize(clusterConfig, customConfig, "hdfs-site", "dfs.webhdfs.enabled", "false");
+
+ ClusterTopology topology = createNiceMock(ClusterTopology.class);
+ Stack stack = createNiceMock(Stack.class);
+ Collection<String> services = ImmutableList.of("HDFS");
+ expect(stack.getServices()).andReturn(services).anyTimes();
+ expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
+ expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes();
+ expect(topology.getHostGroupInfo()).andReturn(emptyMap()).anyTimes();
+ replay(stack, topology);
+
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+ updater.applyTypeSpecificFilter(BlueprintExportType.MINIMAL, clusterConfig, stackConfig, services);
+
+ assertEquals(customConfig.getProperties(), clusterConfig.getProperties());
+ }
+
+ private static Configuration createTestStack() {
+ Configuration stackConfig = Configuration.newEmpty();
+ stackConfig.setProperty("core-site", "io.file.buffer.size", "131072");
+ stackConfig.setProperty("core-site", "hadoop.security.authorization", "false");
+ stackConfig.setProperty("core-site", "fs.trash.interval", "360");
+ stackConfig.setProperty("hdfs-site", "dfs.namenode.name.dir", "/hadoop/hdfs/namenode");
+ stackConfig.setProperty("hdfs-site", "dfs.datanode.data.dir", "/hadoop/hdfs/data");
+ stackConfig.setProperty("hdfs-site", "dfs.webhdfs.enabled", "true");
+ return stackConfig;
+ }
+
+ private static void customize(Configuration clusterConfig, Configuration customConfig, String configType, String propertyName, String value) {
+ clusterConfig.setProperty(configType, propertyName, value);
+ customConfig.setProperty(configType, propertyName, value);
+ }
private static String createExportedAddress(String expectedPortNum, String expectedHostGroupName) {
return createExportedHostName(expectedHostGroupName, expectedPortNum);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
index e1f5583..cfd8fc8 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
@@ -10,8 +10,7 @@
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distribut
- * ed on an "AS IS" BASIS,
+ * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
@@ -23,66 +22,40 @@ import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.ambari.server.api.util.TreeNode;
import org.apache.ambari.server.api.util.TreeNodeImpl;
import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.StackLevelConfigurationRequest;
-import org.apache.ambari.server.controller.StackServiceRequest;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.HostGroup;
import org.apache.ambari.server.topology.HostGroupInfo;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
/**
* ExportBlueprintRequest unit tests.
*/
-@SuppressWarnings("unchecked")
public class ExportBlueprintRequestTest {
private static final String CLUSTER_NAME = "c1";
private static final String CLUSTER_ID = "2";
- private AmbariManagementController controller = createNiceMock(AmbariManagementController.class);
-
- @Before
- public void setupTest() throws Exception {
- Field f = ExportBlueprintRequest.class.getDeclaredField("controller");
- f.setAccessible(true);
- f.set(null, controller);
-
- expect(controller.getStackServices((Set<StackServiceRequest>) anyObject())).andReturn(
- Collections.emptySet()).anyTimes();
- expect(controller.getStackLevelConfigurations((Set<StackLevelConfigurationRequest>) anyObject())).andReturn(
- Collections.emptySet()).anyTimes();
-
- replay(controller);
- }
-
- @After
- public void tearDown() {
- reset(controller);
- }
-
- //todo: test configuration processing
-
@Test
public void testExport_noConfigs() throws Exception {
+ AmbariManagementController controller = createNiceMock(AmbariManagementController.class);
+ expect(controller.getStackServices(anyObject())).andReturn(Collections.emptySet()).anyTimes();
+ expect(controller.getStackLevelConfigurations(anyObject())).andReturn(Collections.emptySet()).anyTimes();
+ replay(controller);
+
Resource clusterResource = new ResourceImpl(Resource.Type.Cluster);
clusterResource.setProperty(ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME);
clusterResource.setProperty(ClusterResourceProvider.CLUSTER_ID_PROPERTY_ID, CLUSTER_ID);
@@ -116,7 +89,7 @@ public class ExportBlueprintRequestTest {
processHostGroupComponents(host3Node, host3ComponentsList);
// test
- ExportBlueprintRequest exportBlueprintRequest = new ExportBlueprintRequest(clusterNode);
+ ExportBlueprintRequest exportBlueprintRequest = new ExportBlueprintRequest(clusterNode, controller);
// assertions
assertEquals(CLUSTER_NAME, exportBlueprintRequest.getClusterName());
@@ -163,4 +136,5 @@ public class ExportBlueprintRequestTest {
hostComponentsNode.addChild(componentResource, "host_component_" + componentCount++);
}
}
+
}