You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/11/16 14:39:34 UTC
[04/31] ambari git commit: AMBARI-18355: Conditional dependencies in
stack defition to handle blueprint validation gracefully (Amruta Borkar via
dili)
AMBARI-18355: Conditional dependencies in stack defition to handle blueprint validation gracefully (Amruta Borkar via dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bc7a29f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bc7a29f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bc7a29f5
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: bc7a29f54046978b79f659d94a17f6335f693191
Parents: a190600
Author: Di Li <di...@apache.org>
Authored: Mon Nov 14 15:33:36 2016 -0500
Committer: Di Li <di...@apache.org>
Committed: Mon Nov 14 15:33:36 2016 -0500
----------------------------------------------------------------------
.../server/state/DependencyConditionInfo.java | 104 +++++++++++++++++++
.../ambari/server/state/DependencyInfo.java | 37 ++++++-
.../server/topology/BlueprintValidatorImpl.java | 18 +++-
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 43 +++++++-
.../server/topology/BlueprintImplTest.java | 41 --------
.../topology/BlueprintValidatorImplTest.java | 75 ++++++++++++-
6 files changed, 270 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
new file mode 100644
index 0000000..66c3e2b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state;
+
+import java.util.Map;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlTransient;
+
+/**
+ * Represents stack component dependency condition information provided in
+ * metainfo.xml file.
+ */
+@XmlTransient
+@XmlSeeAlso({ PropertyExists.class, PropertyValueEquals.class })
+public interface DependencyConditionInfo {
+ /**
+ * Returns true if the dependency condition is satisfied
+ *
+ * @param properties
+ * @return boolean
+ */
+ boolean isResolved(Map<String, Map<String, String>> properties);
+}
+
+/**
+ * This class represents a dependency condition that is based on existence of a
+ * certain property.
+ */
+class PropertyExists implements DependencyConditionInfo {
+ /**
+ * configType of the conditional dependency
+ */
+ protected String configType;
+
+ /**
+ * property of the conditional dependency
+ */
+ protected String property;
+
+ @XmlElement
+ public String getProperty() {
+ return property;
+ }
+
+ public void setProperty(String property) {
+ this.property = property;
+ }
+
+ @XmlElement
+ public String getConfigType() {
+ return configType;
+ }
+
+ public void setConfigType(String configType) {
+ this.configType = configType;
+ }
+
+ @Override
+ public boolean isResolved(Map<String, Map<String, String>> properties) {
+ return (properties.get(configType).containsKey(property));
+ }
+}
+
+/**
+ * This class represents a dependency condition that is based on value of a
+ * certain property.
+ */
+class PropertyValueEquals extends PropertyExists {
+ /**
+ * propertyValue of the property
+ */
+ protected String propertyValue;
+
+ @XmlElement
+ public String getPropertyValue() {
+ return propertyValue;
+ }
+
+ public void setPropertyValue(String propertyValue) {
+ this.propertyValue = propertyValue;
+ }
+
+ @Override
+ public boolean isResolved(Map<String, Map<String, String>> properties) {
+ return (super.isResolved(properties) && propertyValue.equals(properties.get(configType).get(property)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
index e3db662..540767c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
@@ -19,7 +19,12 @@
package org.apache.ambari.server.state;
+import java.util.ArrayList;
+import java.util.List;
import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlElements;
+import org.apache.commons.collections.CollectionUtils;
/**
* Represents stack component dependency information.
@@ -54,7 +59,10 @@ public class DependencyInfo {
@XmlElement(name="auto-deploy")
private AutoDeployInfo m_autoDeploy;
-
+ /**
+ * Conditions for Component dependency to other components.
+ */
+ private List<DependencyConditionInfo> dependencyConditions = new ArrayList<DependencyConditionInfo>();
/**
* Setter for name property.
*
@@ -135,6 +143,33 @@ public class DependencyInfo {
public String getServiceName() {
return serviceName;
}
+ /**
+ * Get the dependencyConditions list
+ *
+ * @return dependencyConditions
+ */
+ @XmlElementWrapper(name="conditions")
+ @XmlElements(@XmlElement(name="condition"))
+ public List<DependencyConditionInfo> getDependencyConditions() {
+ return dependencyConditions;
+ }
+
+ /**
+ * Set dependencyConditions
+ *
+ * @param dependencyConditions
+ */
+ public void setDependencyConditions(List<DependencyConditionInfo> dependencyConditions) {
+ this.dependencyConditions = dependencyConditions;
+ }
+
+ /**
+ * Confirms if dependency have any condition or not
+ * @return true if dependencies are based on a condition
+ */
+ public boolean hasDependencyConditions(){
+ return !CollectionUtils.isEmpty(dependencyConditions);
+ }
@Override
public String toString() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index a5f33ff..c5647c3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -26,10 +26,10 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-import java.util.regex.Pattern;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
import org.apache.ambari.server.state.DependencyInfo;
import org.apache.ambari.server.utils.SecretReference;
import org.apache.ambari.server.utils.VersionUtils;
@@ -135,9 +135,6 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
}
}
if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) {
- if(!hostGroup.getComponentNames().contains("ZKFC")){
- throw new InvalidTopologyException("Compoenent ZKFC is mandatory for hostgroup " + hostGroup+" when NAMENODE HA is enabled");
- }
Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env");
if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) {
ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>( blueprint.getHostGroupsForComponent(component));
@@ -289,6 +286,19 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy();
boolean resolved = false;
+ //check if conditions are met, if any
+ if(dependency.hasDependencyConditions()) {
+ boolean conditionsSatisfied = true;
+ for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions()) {
+ if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties())) {
+ conditionsSatisfied = false;
+ break;
+ }
+ }
+ if(!conditionsSatisfied){
+ continue;
+ }
+ }
if (dependencyScope.equals("cluster")) {
Collection<String> missingDependencyInfo = verifyComponentCardinalityCount(
componentName, new Cardinality("1+"), autoDeployInfo);
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 9c1387d..2b7a684 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -15,7 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<metainfo>
+<metainfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<schemaVersion>2.0</schemaVersion>
<services>
<service>
@@ -32,6 +32,47 @@
<cardinality>1-2</cardinality>
<versionAdvertised>true</versionAdvertised>
<reassignAllowed>true</reassignAllowed>
+ <dependencies>
+ <dependency>
+ <name>HDFS/ZKFC</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ <conditions>
+ <condition xsi:type="propertyExists">
+ <configType>hdfs-site</configType>
+ <property>dfs.nameservices</property>
+ </condition>
+ </conditions>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ <conditions>
+ <condition xsi:type="propertyExists">
+ <configType>hdfs-site</configType>
+ <property>dfs.nameservices</property>
+ </condition>
+ </conditions>
+ </dependency>
+ <dependency>
+ <name>HDFS/JOURNALNODE</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ <conditions>
+ <condition xsi:type="propertyExists">
+ <configType>hdfs-site</configType>
+ <property>dfs.nameservices</property>
+ </condition>
+ </conditions>
+ </dependency>
+ </dependencies>
<commandScript>
<script>scripts/namenode.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index ff9af17..6763b7c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -212,47 +212,6 @@ public class BlueprintImplTest {
assertTrue(entity.getSecurityType() == SecurityType.NONE);
assertTrue(entity.getSecurityDescriptorReference() == null);
}
- @Test(expected = InvalidTopologyException.class)
- public void testValidateConfigurations__hostGroupConfigForNameNodeHANOZKFC() throws Exception {
- Map<String, Map<String, String>> group2Props = new HashMap<>();
- Map<String, String> group2Category2Props = new HashMap<>();
- group2Props.put("category2", group2Category2Props);
- group2Category2Props.put("prop2", "val");
- // set config for group2 which contains a required property
- Configuration group2Configuration = new Configuration(group2Props, EMPTY_ATTRIBUTES, configuration);
- expect(group2.getConfiguration()).andReturn(group2Configuration).atLeastOnce();
-
- expect(group1.getCardinality()).andReturn("1").atLeastOnce();
- expect(group1.getComponents()).andReturn(Arrays.asList(new Component("NAMENODE"),new Component("ZKFC"))).atLeastOnce();
- expect(group2.getCardinality()).andReturn("1").atLeastOnce();
- expect(group2.getComponents()).andReturn(Arrays.asList(new Component("NAMENODE"),new Component("ZKFC"))).atLeastOnce();
- Map<String, String> category2Props = new HashMap<>();
- properties.put("category2", category2Props);
- category2Props.put("prop2", "val");
- group1Components.add("NAMENODE");
- group1Components.add("ZKFC");
- group2Components.add("NAMENODE");
- expect(stack.getServiceForComponent("NAMENODE")).andReturn("SERVICE2").atLeastOnce();
- expect(stack.getServiceForComponent("ZKFC")).andReturn("SERVICE2").atLeastOnce();
- Map<String, String> hdfsProps = new HashMap<String, String>();
- properties.put("hdfs-site", hdfsProps);
- hdfsProps.put("foo", "val");
- hdfsProps.put("bar", "val");
- hdfsProps.put("dfs.nameservices", "val");
- Map<String, String> hadoopProps = new HashMap<String, String>();
- properties.put("hadoop-env", hadoopProps);
- hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group1%");
- hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%");
- replay(stack, group1, group2);
-
- Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
- blueprint.validateRequiredProperties();
- BlueprintEntity entity = blueprint.toEntity();
-
- verify(stack, group1, group2);
- assertTrue(entity.getSecurityType() == SecurityType.NONE);
- assertTrue(entity.getSecurityDescriptorReference() == null);
- }
@Test(expected= IllegalArgumentException.class)
public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHostGroups() throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index b1de8ef..1501c53 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -34,7 +34,9 @@ import java.util.Map;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
import org.apache.ambari.server.state.DependencyInfo;
+import org.easymock.EasyMock;
import org.easymock.EasyMockRule;
import org.easymock.Mock;
import org.easymock.MockType;
@@ -66,15 +68,22 @@ public class BlueprintValidatorImplTest {
@Mock(type = MockType.NICE)
private DependencyInfo dependency1;
+ @Mock(type = MockType.NICE)
+ private DependencyInfo dependency2;
@Mock(type = MockType.NICE)
private ComponentInfo dependencyComponentInfo;
+ @Mock(type = MockType.NICE)
+ private DependencyConditionInfo dependencyConditionInfo1;
+ @Mock(type = MockType.NICE)
+ private DependencyConditionInfo dependencyConditionInfo2;
private final Collection<String> group1Components = new ArrayList<String>();
private final Collection<String> group2Components = new ArrayList<String>();
private final Collection<String> services = new ArrayList<String>();
private Collection<DependencyInfo> dependencies1 = new ArrayList<DependencyInfo>();
+ private List<DependencyConditionInfo> dependenciesConditionInfos1 = new ArrayList<DependencyConditionInfo>();
private AutoDeployInfo autoDeploy = new AutoDeployInfo();
private Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>();
private Configuration configuration = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
@@ -105,13 +114,15 @@ public class BlueprintValidatorImplTest {
expect(stack.getCardinality("component1")).andReturn(new Cardinality("1"));
expect(stack.getCardinality("component2")).andReturn(new Cardinality("1+"));
expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+"));
+ dependenciesConditionInfos1.add(dependencyConditionInfo1);
+ dependenciesConditionInfos1.add(dependencyConditionInfo2);
expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes();
}
@After
public void tearDown() {
- reset(blueprint, stack, group1, group2, dependency1);
+ reset(blueprint, stack, group1, group2, dependency1, dependency2, dependencyConditionInfo1, dependencyConditionInfo2);
}
@Test
@@ -341,4 +352,66 @@ public class BlueprintValidatorImplTest {
verify(group1);
}
+ @Test(expected=InvalidTopologyException.class)
+ public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIsSatisfied() throws Exception {
+ // GIVEN
+ hostGroups.clear();
+ hostGroups.put("group1", group1);
+
+ group1Components.add("component-1");
+ dependencies1.add(dependency1);
+ dependencies1.add(dependency2);
+ services.addAll(Collections.singleton("service-1"));
+
+
+ expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes();
+ expect(blueprint.getName()).andReturn("blueprint-1").anyTimes();
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProps = new HashMap<String, String>();
+ typeProps.put("yarn.resourcemanager.hostname", "testhost");
+ properties.put("yarn-site", typeProps);
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Cardinality cardinality = new Cardinality("1");
+
+ expect(stack.getComponents("service-1")).andReturn(Arrays.asList("component-1")).anyTimes();
+ expect(stack.getAutoDeployInfo("component-1")).andReturn(autoDeploy).anyTimes();
+ expect(stack.getDependenciesForComponent("component-1")).andReturn(dependencies1).anyTimes();
+ expect(stack.getCardinality("component-1")).andReturn(cardinality).anyTimes();
+
+ AutoDeployInfo dependencyAutoDeploy = null;
+
+ expect(dependency1.getScope()).andReturn("host").anyTimes();
+ expect(dependency1.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+ expect(dependency1.getComponentName()).andReturn("component-d").anyTimes();
+ expect(dependency1.getServiceName()).andReturn("service-d").anyTimes();
+ expect(dependency1.getName()).andReturn("dependency-1").anyTimes();
+ expect(dependency1.hasDependencyConditions()).andReturn(true).anyTimes();
+ expect(dependency1.getDependencyConditions()).andReturn(dependenciesConditionInfos1).anyTimes();
+ expect(dependency2.getScope()).andReturn("host").anyTimes();
+ expect(dependency2.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+ expect(dependency2.getComponentName()).andReturn("component-d").anyTimes();
+ expect(dependency2.getServiceName()).andReturn("service-d").anyTimes();
+ expect(dependency2.getName()).andReturn("dependency-2").anyTimes();
+ expect(dependency2.hasDependencyConditions()).andReturn(false).anyTimes();
+
+ expect(dependencyConditionInfo1.isResolved(EasyMock.anyObject(Map.class))).andReturn(true).anyTimes();
+ expect(dependencyConditionInfo2.isResolved(EasyMock.anyObject(Map.class))).andReturn(false).anyTimes();
+
+
+ expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes();
+ expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes();
+
+ replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2);
+
+ // WHEN
+ BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
+ validator.validateTopology();
+
+ // THEN
+ verify(group1);
+
+ }
}