You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/05/12 14:36:48 UTC
[09/26] ambari git commit: AMBARI-20872 Required properties for
services in the blueprint are validated before the cluster provisioning is
started (when the cluster creation template is posted)
AMBARI-20872 Required properties for services in the blueprint are validated before the cluster provisioning is started (when the cluster creation template is posted)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3edbc2c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3edbc2c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3edbc2c4
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 3edbc2c41d489c20d1f503723491461eaa328fab
Parents: 55750b9
Author: lpuskas <lp...@apache.org>
Authored: Wed May 3 18:31:40 2017 +0200
Committer: lpuskas <lp...@apache.org>
Committed: Thu May 11 13:35:40 2017 +0200
----------------------------------------------------------------------
.../server/topology/BlueprintValidatorImpl.java | 88 ++----
.../RequiredConfigPropertiesValidator.java | 188 ++++++++++++
.../validators/TopologyValidatorFactory.java | 3 +-
.../server/topology/BlueprintImplTest.java | 13 -
.../RequiredConfigPropertiesValidatorTest.java | 302 +++++++++++++++++++
5 files changed, 512 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3edbc2c4/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 9688c60..2e5995c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -20,7 +20,6 @@ package org.apache.ambari.server.topology;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -49,6 +48,7 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
this.blueprint = blueprint;
this.stack = blueprint.getStack();
}
+
@Override
public void validateTopology() throws InvalidTopologyException {
LOGGER.info("Validating topology for blueprint: [{}]", blueprint.getName());
@@ -57,7 +57,7 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
for (HostGroup group : hostGroups) {
Map<String, Collection<DependencyInfo>> missingGroupDependencies = validateHostGroup(group);
- if (! missingGroupDependencies.isEmpty()) {
+ if (!missingGroupDependencies.isEmpty()) {
missingDependencies.put(group.getName(), missingGroupDependencies);
}
}
@@ -73,27 +73,24 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
cardinalityFailures.addAll(verifyComponentInAllHostGroups(component, autoDeploy));
} else {
cardinalityFailures.addAll(verifyComponentCardinalityCount(
- component, cardinality, autoDeploy));
+ component, cardinality, autoDeploy));
}
}
}
- if (! missingDependencies.isEmpty() || ! cardinalityFailures.isEmpty()) {
+ if (!missingDependencies.isEmpty() || !cardinalityFailures.isEmpty()) {
generateInvalidTopologyException(missingDependencies, cardinalityFailures);
}
}
@Override
public void validateRequiredProperties() throws InvalidTopologyException {
- //todo: combine with RequiredPasswordValidator
- Map<String, Map<String, Collection<String>>> missingProperties =
- new HashMap<>();
// we don't want to include default stack properties so we can't just use hostGroup full properties
Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
// we need to have real passwords, not references
- if(clusterConfigurations != null) {
+ if (clusterConfigurations != null) {
StringBuilder errorMessage = new StringBuilder();
boolean containsSecretReferences = false;
for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) {
@@ -104,16 +101,16 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
String propertyValue = propertyEntry.getValue();
if (propertyValue != null) {
if (SecretReference.isSecret(propertyValue)) {
- errorMessage.append(" Config:").append(configType).append(" Property:").append(propertyName).append("\n");
+ errorMessage.append(" Config:" + configType + " Property:" + propertyName + "\n");
containsSecretReferences = true;
}
}
}
}
}
- if(containsSecretReferences) {
+ if (containsSecretReferences) {
throw new InvalidTopologyException("Secret references are not allowed in blueprints, " +
- "replace following properties with real passwords:\n"+errorMessage.toString());
+ "replace following properties with real passwords:\n" + errorMessage.toString());
}
}
@@ -129,9 +126,9 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
if (component.equals("MYSQL_SERVER")) {
Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env");
if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null
- && hiveEnvConfig.get("hive_database").startsWith("Existing")) {
+ && hiveEnvConfig.get("hive_database").startsWith("Existing")) {
throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" +
- " using existing db!");
+ " using existing db!");
}
}
if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) {
@@ -162,72 +159,27 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
if (component.equals("HIVE_METASTORE")) {
Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env");
- if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") !=null
- && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database")
- && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
- && stack.getName().equalsIgnoreCase("HDP")) {
+ if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null
+ && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database")
+ && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
+ && stack.getName().equalsIgnoreCase("HDP")) {
throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " +
- "and repo version 2.3.2+!");
+ "and repo version 2.3.2+!");
}
}
if (component.equals("OOZIE_SERVER")) {
Map<String, String> oozieEnvConfig = clusterConfigurations.get("oozie-env");
- if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") !=null
- && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database")
- && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
- && stack.getName().equalsIgnoreCase("HDP")) {
+ if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") != null
+ && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database")
+ && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0
+ && stack.getName().equalsIgnoreCase("HDP")) {
throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " +
- "and repo version 2.3.2+!");
+ "and repo version 2.3.2+!");
}
}
-
- //for now, AMBARI is not recognized as a service in Stacks
- if (! component.equals("AMBARI_SERVER")) {
- String serviceName = stack.getServiceForComponent(component);
- if (processedServices.add(serviceName)) {
- Collection<Stack.ConfigProperty> requiredServiceConfigs =
- stack.getRequiredConfigurationProperties(serviceName);
-
- for (Stack.ConfigProperty requiredConfig : requiredServiceConfigs) {
- String configCategory = requiredConfig.getType();
- String propertyName = requiredConfig.getName();
- if (! stack.isPasswordProperty(serviceName, configCategory, propertyName)) {
- Collection<String> typeRequirements = allRequiredProperties.get(configCategory);
- if (typeRequirements == null) {
- typeRequirements = new HashSet<>();
- allRequiredProperties.put(configCategory, typeRequirements);
- }
- typeRequirements.add(propertyName);
- }
- }
- }
- }
- }
- for (Map.Entry<String, Collection<String>> requiredTypeProperties : allRequiredProperties.entrySet()) {
- String requiredCategory = requiredTypeProperties.getKey();
- Collection<String> requiredProperties = requiredTypeProperties.getValue();
- Collection<String> operationalTypeProps = operationalConfiguration.containsKey(requiredCategory) ?
- operationalConfiguration.get(requiredCategory).keySet() :
- Collections.<String>emptyList();
-
- requiredProperties.removeAll(operationalTypeProps);
- if (! requiredProperties.isEmpty()) {
- String hostGroupName = hostGroup.getName();
- Map<String, Collection<String>> hostGroupMissingProps = missingProperties.get(hostGroupName);
- if (hostGroupMissingProps == null) {
- hostGroupMissingProps = new HashMap<>();
- missingProperties.put(hostGroupName, hostGroupMissingProps);
- }
- hostGroupMissingProps.put(requiredCategory, requiredProperties);
- }
}
}
-
- if (! missingProperties.isEmpty()) {
- throw new InvalidTopologyException("Missing required properties. Specify a value for these " +
- "properties in the blueprint configuration. " + missingProperties);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/3edbc2c4/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
new file mode 100644
index 0000000..759d9e9
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology.validators;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.HostGroup;
+import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.TopologyValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Validates the configuration by checking the existence of required properties for the services listed in the blueprint.
+ * Required properties are specified in the stack and are tied to config types and services.
+ *
+ * The validator ignores password properties that should never be specified in the artifacts (blueprint / cluster creation template)
+ */
+public class RequiredConfigPropertiesValidator implements TopologyValidator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RequiredConfigPropertiesValidator.class);
+
+ /**
+ * Validates the configuration coming from the blueprint and cluster creation template and ensures that all the required properties are provided.
+ * It's expected, that a in hostrgroup containing components for a given service all required configuration for the given service is available.
+ *
+ * @param topology the topology instance holding the configuration for cluster provisioning
+ * @throws InvalidTopologyException when there are missing configuration types or properties related to services in the blueprint
+ */
+ @Override
+ public void validate(ClusterTopology topology) throws InvalidTopologyException {
+
+ // collect required properties
+ Map<String, Map<String, Collection<String>>> requiredPropertiesByService = getRequiredPropertiesByService(topology.getBlueprint());
+
+ // find missing properties in the cluster configuration
+ Map<String, Collection<String>> missingProperties = new HashMap<>();
+ Map<String, Map<String, String>> topologyConfiguration = new HashMap<>(topology.getConfiguration().getFullProperties(1));
+
+ for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) {
+ LOGGER.debug("Processing hostgroup configurations for hostgroup: {}", hostGroup.getName());
+
+ // copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig
+ Map<String, Map<String, String>> operationalConfigurations = new HashMap<>(topologyConfiguration);
+
+ for (Map.Entry<String, Map<String, String>> hostgroupConfigEntry : hostGroup.getConfiguration().getProperties().entrySet()) {
+ if (operationalConfigurations.containsKey(hostgroupConfigEntry.getKey())) {
+ operationalConfigurations.get(hostgroupConfigEntry.getKey()).putAll(hostgroupConfigEntry.getValue());
+ } else {
+ operationalConfigurations.put(hostgroupConfigEntry.getKey(), hostgroupConfigEntry.getValue());
+ }
+ }
+
+ for (String hostGroupService : hostGroup.getServices()) {
+
+ if (!requiredPropertiesByService.containsKey(hostGroupService)) {
+ // there are no required properties for the service
+ LOGGER.debug("There are no required properties found for hostgroup/service: [{}/{}]", hostGroup.getName(), hostGroupService);
+ continue;
+ }
+
+ Map<String, Collection<String>> requiredPropertiesByType = requiredPropertiesByService.get(hostGroupService);
+
+ for (String configType : requiredPropertiesByType.keySet()) {
+
+ // We need a copy not to modify the original
+ Collection<String> requiredPropertiesForType = new HashSet(requiredPropertiesByType.get(configType));
+
+ if (!operationalConfigurations.containsKey(configType)) {
+ // all required configuration is missing for the config type
+ missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType);
+ continue;
+ }
+
+ Collection<String> operationalConfigsForType = operationalConfigurations.get(configType).keySet();
+ requiredPropertiesForType.removeAll(operationalConfigsForType);
+ if (!requiredPropertiesForType.isEmpty()) {
+ LOGGER.info("Found missing properties in hostgroup: {}, config type: {}, mising properties: {}", hostGroup.getName(),
+ configType, requiredPropertiesForType);
+ missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType);
+ }
+ }
+ }
+
+ }
+
+ if (!missingProperties.isEmpty()) {
+ throw new InvalidTopologyException("Missing required properties. Specify a value for these " +
+ "properties in the blueprint or cluster creation template configuration. " + missingProperties);
+ }
+
+ }
+
+
+ /**
+ * Collects required properties for services in the blueprint. Configuration properties are returned by configuration type.
+ * service -> configType -> properties
+ *
+ * @param blueprint the blueprint from the cluster topology
+ * @return a map with configuration types mapped to collections of required property names
+ */
+
+ private Map<String, Map<String, Collection<String>>> getRequiredPropertiesByService(Blueprint blueprint) {
+
+ Map<String, Map<String, Collection<String>>> requiredPropertiesForServiceByType = new HashMap<>();
+
+ for (String bpService : blueprint.getServices()) {
+ LOGGER.debug("Collecting required properties for the service: {}", bpService);
+
+ Collection<Stack.ConfigProperty> requiredConfigsForService = blueprint.getStack().getRequiredConfigurationProperties(bpService);
+ Map<String, Collection<String>> requiredPropertiesByConfigType = new HashMap<>();
+
+ for (Stack.ConfigProperty configProperty : requiredConfigsForService) {
+
+ if (configProperty.getPropertyTypes() != null && configProperty.getPropertyTypes().contains(PropertyInfo.PropertyType.PASSWORD)) {
+ LOGGER.debug("Skipping required property validation for password type: {}", configProperty.getName());
+ // skip password types
+ continue;
+ }
+
+ // add / get service related required propeByType map
+ if (requiredPropertiesForServiceByType.containsKey(bpService)) {
+ requiredPropertiesByConfigType = requiredPropertiesForServiceByType.get(bpService);
+ } else {
+ LOGGER.debug("Adding required properties entry for service: {}", bpService);
+ requiredPropertiesForServiceByType.put(bpService, requiredPropertiesByConfigType);
+ }
+
+ // add collection of required properties
+ Collection<String> requiredPropsForType = new HashSet<>();
+ if (requiredPropertiesByConfigType.containsKey(configProperty.getType())) {
+ requiredPropsForType = requiredPropertiesByConfigType.get(configProperty.getType());
+ } else {
+ LOGGER.debug("Adding required properties entry for configuration type: {}", configProperty.getType());
+ requiredPropertiesByConfigType.put(configProperty.getType(), requiredPropsForType);
+ }
+
+ requiredPropsForType.add(configProperty.getName());
+ LOGGER.debug("Added required property for service; {}, configuration type: {}, property: {}", bpService,
+ configProperty.getType(), configProperty.getName());
+ }
+ }
+
+ LOGGER.info("Identified required properties for blueprint services: {}", requiredPropertiesForServiceByType);
+ return requiredPropertiesForServiceByType;
+
+ }
+
+ private Map<String, Collection<String>> addTomissingProperties(Map<String, Collection<String>> missingProperties, String hostGroup, Collection<String> values) {
+ Map<String, Collection<String>> missing;
+
+ if (missingProperties == null) {
+ missing = new HashMap<>();
+ } else {
+ missing = new HashMap<>(missingProperties);
+ }
+
+ if (!missing.containsKey(hostGroup)) {
+ missing.put(hostGroup, new HashSet<String>());
+ }
+
+ missing.get(hostGroup).addAll(values);
+
+ return missing;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3edbc2c4/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java
index 0e77301..5a6f64e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java
@@ -24,7 +24,8 @@ public class TopologyValidatorFactory {
List<TopologyValidator> validators;
public TopologyValidatorFactory() {
- validators = ImmutableList.of(new RequiredPasswordValidator(), new HiveServiceValidator(), new StackConfigTypeValidator());
+ validators = ImmutableList.of(new RequiredConfigPropertiesValidator(), new RequiredPasswordValidator(), new HiveServiceValidator(),
+ new StackConfigTypeValidator());
}
public TopologyValidator createConfigurationValidatorChain() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/3edbc2c4/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index 255145c..5f4e00b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -128,16 +128,6 @@ public class BlueprintImplTest {
assertTrue(entity.getSecurityDescriptorReference().equals("testRef"));
}
- @Test(expected = InvalidTopologyException.class)
- public void testValidateConfigurations__basic_negative() throws Exception {
- expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
- replay(stack, group1, group2);
-
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
- blueprint.validateRequiredProperties();
- verify(stack, group1, group2);
- }
-
@Test
public void testValidateConfigurations__hostGroupConfig() throws Exception {
Map<String, Map<String, String>> group2Props = new HashMap<>();
@@ -157,7 +147,6 @@ public class BlueprintImplTest {
category2Props.put("prop2", "val");
group1Components.add("NAMENODE");
group2Components.add("NAMENODE");
- expect(stack.getServiceForComponent("NAMENODE")).andReturn("SERVICE2").atLeastOnce();
Map<String, String> hdfsProps = new HashMap<>();
properties.put("hdfs-site", hdfsProps);
hdfsProps.put("foo", "val");
@@ -195,8 +184,6 @@ public class BlueprintImplTest {
group1Components.add("ZKFC");
group2Components.add("NAMENODE");
group2Components.add("ZKFC");
- expect(stack.getServiceForComponent("NAMENODE")).andReturn("SERVICE2").atLeastOnce();
- expect(stack.getServiceForComponent("ZKFC")).andReturn("SERVICE2").atLeastOnce();
Map<String, String> hdfsProps = new HashMap<>();
properties.put("hdfs-site", hdfsProps);
hdfsProps.put("foo", "val");
http://git-wip-us.apache.org/repos/asf/ambari/blob/3edbc2c4/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java
new file mode 100644
index 0000000..8ead623
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology.validators;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.HostGroup;
+import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockRule;
+import org.easymock.EasyMockSupport;
+import org.easymock.Mock;
+import org.easymock.TestSubject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class RequiredConfigPropertiesValidatorTest extends EasyMockSupport {
+
+ @Rule
+ public EasyMockRule mocks = new EasyMockRule(this);
+
+ @Mock
+ private ClusterTopology clusterTopologyMock;
+
+ @Mock
+ private Configuration topologyConfigurationMock;
+
+ @Mock
+ private Blueprint blueprintMock;
+
+ @Mock
+ private Stack stackMock;
+
+ @Mock
+ private HostGroup slaveHostGroupMock;
+
+ @Mock
+ private HostGroup masterHostGroupMock;
+
+ @Mock
+ private Configuration slaveHostGroupConfigurationMock;
+
+ @Mock
+ private Configuration masterHostGroupConfigurationMock;
+
+
+ private Map<String, Map<String, String>> topologyConfigurationMap = new HashMap<>();
+ private Map<String, Map<String, String>> masterHostGroupConfigurationMap = new HashMap<>();
+ private Map<String, Map<String, String>> slaveHostGroupConfigurationMap = new HashMap<>();
+ private Collection<String> bpServices = new HashSet<>();
+ private Collection<String> slaveHostGroupServices = new HashSet<>();
+ private Collection<String> masterHostGroupServices = new HashSet<>();
+ private Map<String, HostGroup> hostGroups = new HashMap<>();
+ private Map<String, Collection<String>> missingProps = new HashMap<>();
+
+ @TestSubject
+ private RequiredConfigPropertiesValidator testSubject = new RequiredConfigPropertiesValidator();
+
+ /**
+ * Assembles the basic default fixture of the test:
+ * The blueprint has 2 hostgroups, both of them contains the KERBEROS service (or KERBEROS_CLIENT)
+ * The changing items are the configurations, that my come from the topology (bp + cct) or the hostg roups
+ */
+
+ @Before
+ public void setup() {
+ resetAll();
+
+ EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(topologyConfigurationMock);
+ EasyMock.expect(topologyConfigurationMock.getFullProperties(1)).andReturn(topologyConfigurationMap);
+
+ EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes();
+
+ EasyMock.expect(blueprintMock.getHostGroups()).andReturn(hostGroups);
+ EasyMock.expect(blueprintMock.getServices()).andReturn(bpServices);
+ EasyMock.expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes();
+
+ EasyMock.expect(masterHostGroupMock.getName()).andReturn("master").anyTimes();
+ EasyMock.expect(masterHostGroupMock.getConfiguration()).andReturn(masterHostGroupConfigurationMock).anyTimes();
+ EasyMock.expect(masterHostGroupMock.getServices()).andReturn(masterHostGroupServices);
+
+
+ EasyMock.expect(slaveHostGroupMock.getName()).andReturn("slave").anyTimes();
+ EasyMock.expect(slaveHostGroupMock.getConfiguration()).andReturn(slaveHostGroupConfigurationMock).anyTimes();
+ EasyMock.expect(slaveHostGroupMock.getServices()).andReturn(slaveHostGroupServices);
+
+ // there are 2 hostgroups to be considered by the test
+ hostGroups.put("master", masterHostGroupMock);
+ hostGroups.put("slave", slaveHostGroupMock);
+
+ // services in the blueprint
+ bpServices.addAll(Arrays.asList("KERBEROS", "OOZIE"));
+
+ // host group services
+ masterHostGroupServices.addAll(Arrays.asList("KERBEROS"));
+ slaveHostGroupServices.addAll(Arrays.asList("KERBEROS"));
+
+ EasyMock.expect(masterHostGroupConfigurationMock.getProperties()).andReturn(masterHostGroupConfigurationMap);
+ EasyMock.expect(slaveHostGroupConfigurationMock.getProperties()).andReturn(slaveHostGroupConfigurationMap);
+
+ // services in the blueprint
+ bpServices.addAll(Arrays.asList("KERBEROS", "OOZIE"));
+
+ // required properties for listed services
+ EasyMock.expect(stackMock.getRequiredConfigurationProperties("KERBEROS")).
+ andReturn(Arrays.asList(
+ new Stack.ConfigProperty("kerberos-env", "realm", "value"),
+ new Stack.ConfigProperty("kerberos-env", "kdc_type", "value"), // this is missing!
+ new Stack.ConfigProperty("krb5-conf", "domains", "smthg")));
+
+ EasyMock.expect(stackMock.getRequiredConfigurationProperties("OOZIE")).andReturn(Collections.EMPTY_LIST);
+
+ }
+
+ @Test
+ public void testShouldValidationFailWhenNoHostGroupConfigurationProvidedAndRequiredConfigTypesAreMissing() throws Exception {
+
+ // GIVEN
+ // all the configuration comes from the bp, cct hg configs are empty
+ topologyConfigurationMap.put("kerberos-env", new HashMap<String, String>());
+ topologyConfigurationMap.get("kerberos-env").put("realm", "etwas");
+ topologyConfigurationMap.get("kerberos-env").put("kdc_type", "mit-kdc");
+
+ // note, that the krb-5 config type is missing! (see the required properties in the fixture!)
+ missingProps.put("slave", Arrays.asList("domains"));
+ missingProps.put("master", Arrays.asList("domains"));
+
+ replayAll();
+
+ // WHEN
+ String expectedMsg = String.format("Missing required properties. Specify a value for these properties in the blueprint or cluster creation template configuration. %s", missingProps);
+ String actualMsg = "";
+ try {
+ testSubject.validate(clusterTopologyMock);
+ } catch (InvalidTopologyException e) {
+ actualMsg = e.getMessage();
+ }
+
+ // THEN
+ // Exception is thrown, as the krb5-conf typeeis not provided
+ Assert.assertEquals("The exception message should be the expected one", expectedMsg, actualMsg);
+ }
+
+ @Test
+ public void testShouldValidationFailWhenNoHostGroupConfigurationProvidedAndRequiredPropertiesAreMissing() throws Exception {
+ // GIVEN
+
+ // configuration from the blueprint / cluster creation template
+ topologyConfigurationMap.put("kerberos-env", new HashMap<String, String>());
+ topologyConfigurationMap.get("kerberos-env").put("realm", "etwas");
+
+ // note, that tehe kdc_type is mssing from the operational config
+
+ topologyConfigurationMap.put("krb5-conf", new HashMap<String, String>());
+ topologyConfigurationMap.get("krb5-conf").put("domains", "smthg");
+
+ missingProps.put("master", Arrays.asList("kdc_type"));
+ missingProps.put("slave", Arrays.asList("kdc_type"));
+
+ replayAll();
+
+ // WHEN
+ String expectedMsg = String.format("Missing required properties. Specify a value for these properties in the blueprint or cluster creation template configuration. %s", missingProps);
+ String actualMsg = "";
+ try {
+ testSubject.validate(clusterTopologyMock);
+ } catch (InvalidTopologyException e) {
+ actualMsg = e.getMessage();
+ }
+
+ // THEN
+ // Exception is thrown, as the krb5-conf typee is not provideds
+ Assert.assertEquals("The exception message should be the expected one", expectedMsg, actualMsg);
+
+ }
+
+
+ @Test
+ public void testShouldValidationFailWhenHostGroupConfigurationProvidedAndRequiredConfigTypesAreMissingFromBothHostgroups() throws Exception {
+ // GIVEN
+ // configuration come in the host groups, there are missing config types in both hostgroups
+
+ missingProps.put("master", Arrays.asList("kdc_type", "domains", "realm"));
+ missingProps.put("slave", Arrays.asList("kdc_type", "domains", "realm"));
+
+ replayAll();
+
+ // WHEN
+ String expectedMsg = String.format("Missing required properties. Specify a value for these properties in the blueprint or cluster creation template configuration. %s", missingProps);
+ String actualMsg = "";
+ try {
+ testSubject.validate(clusterTopologyMock);
+ } catch (InvalidTopologyException e) {
+ actualMsg = e.getMessage();
+ }
+
+ // THEN
+ // Exception is thrown, as the krb5-conf typee is not provideds
+ Assert.assertEquals("The exception message should be the expected one", expectedMsg, actualMsg);
+ }
+
+ @Test
+ public void testShouldValidationFailWhenHostGroupConfigurationProvidedAndRequiredConfigTypesAreMissingFromSlaveHostgroup() throws Exception {
+ // GIVEN
+ // configuration come in the host groups, there are missing config types in both hostgroups
+ masterHostGroupConfigurationMap.put("kerberos-env", new HashMap<String, String>());
+ masterHostGroupConfigurationMap.get("kerberos-env").put("realm", "etwas");
+ masterHostGroupConfigurationMap.get("kerberos-env").put("kdc_type", "mit-kdc");
+ masterHostGroupConfigurationMap.put("krb5-conf", new HashMap<String, String>());
+ masterHostGroupConfigurationMap.get("krb5-conf").put("domains", "smthg");
+
+ missingProps.put("slave", Arrays.asList("kdc_type", "domains", "realm"));
+
+ replayAll();
+
+ // WHEN
+ String expectedMsg = String.format("Missing required properties. Specify a value for these properties in the blueprint or cluster creation template configuration. %s", missingProps);
+ String actualMsg = "";
+ try {
+ testSubject.validate(clusterTopologyMock);
+ } catch (InvalidTopologyException e) {
+ actualMsg = e.getMessage();
+ }
+
+ // THEN
+ // Exception is thrown, as the krb5-conf typee is not provideds
+ Assert.assertEquals("The exception message should be the expected one", expectedMsg, actualMsg);
+ }
+
+ @Test
+ public void testShouldValidationPassWhenAllRequiredPropertiesAreProvidedInHostGroupConfiguration() throws Exception {
+ // GIVEN
+
+ masterHostGroupConfigurationMap.put("kerberos-env", new HashMap<String, String>());
+ masterHostGroupConfigurationMap.get("kerberos-env").put("realm", "etwas");
+ masterHostGroupConfigurationMap.get("kerberos-env").put("kdc_type", "mit-kdc");
+ masterHostGroupConfigurationMap.put("krb5-conf", new HashMap<String, String>());
+ masterHostGroupConfigurationMap.get("krb5-conf").put("domains", "smthg");
+
+ slaveHostGroupConfigurationMap.put("kerberos-env", new HashMap<String, String>());
+ slaveHostGroupConfigurationMap.get("kerberos-env").put("realm", "etwas");
+ slaveHostGroupConfigurationMap.get("kerberos-env").put("kdc_type", "mit-kdc");
+ slaveHostGroupConfigurationMap.put("krb5-conf", new HashMap<String, String>());
+ slaveHostGroupConfigurationMap.get("krb5-conf").put("domains", "smthg");
+
+ replayAll();
+
+ // WHEN
+
+ testSubject.validate(clusterTopologyMock);
+
+ // THEN
+ // no exceptions thrown
+
+ }
+
+
+ @Test
+ public void testShouldValidationPassWhenAllRequiredPropertiesAreProvidedInTopologyConfiguration() throws Exception {
+ // GIVEN
+ // configuration from the blueprint / cluster creation template
+ topologyConfigurationMap.put("kerberos-env", new HashMap<String, String>());
+ topologyConfigurationMap.get("kerberos-env").put("realm", "etwas");
+ topologyConfigurationMap.get("kerberos-env").put("kdc_type", "value");
+
+ topologyConfigurationMap.put("krb5-conf", new HashMap<String, String>());
+ topologyConfigurationMap.get("krb5-conf").put("domains", "smthg");
+
+ replayAll();
+
+ // WHEN
+ testSubject.validate(clusterTopologyMock);
+
+ // THEN
+ // no exceptions thrown
+
+ }
+
+}
\ No newline at end of file