You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ad...@apache.org on 2017/11/24 12:40:17 UTC
[35/39] ambari git commit: AMBARI-22325 Cluster template object
improvements (benyoka)
AMBARI-22325 Cluster template object improvements (benyoka)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3a016f83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3a016f83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3a016f83
Branch: refs/heads/branch-feature-AMBARI-14714-blueprintv2
Commit: 3a016f8380627c916726edb5137e4df1336b527f
Parents: c50ce3d
Author: Balazs Bence Sari <be...@apache.org>
Authored: Tue Nov 21 10:32:23 2017 +0100
Committer: Doroszlai, Attila <ad...@hortonworks.com>
Committed: Fri Nov 24 13:30:46 2017 +0100
----------------------------------------------------------------------
.../ambari/server/topology/ComponentV2.java | 6 +-
.../topology/ProvisionClusterTemplate.java | 72 ++++++++++++++++----
.../ProvisionClusterTemplateFactory.java | 9 +--
.../apache/ambari/server/topology/Service.java | 6 +-
.../ambari/server/topology/ServiceId.java | 27 +++-----
.../server/topology/BlueprintV2FactoryTest.java | 16 ++---
.../topology/ProvisionClusterTemplateTest.java | 49 ++++++++++++-
.../blueprintv2/cluster_template_v2.json | 15 +++-
.../cluster_template_v2_invalid_hostgroup.json | 70 +++++++++++++++++++
9 files changed, 215 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
index 0d26ef6..42158f4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
@@ -30,7 +30,7 @@ public class ComponentV2 implements Configurable {
private String name;
- private ServiceId serviceId = new ServiceId();
+ private ServiceId serviceId = new ServiceId(null, null);
private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START;
@@ -110,12 +110,12 @@ public class ComponentV2 implements Configurable {
@JsonProperty("service_group")
public void setServiceGroup(String serviceGroup) {
- serviceId.setServiceGroup(serviceGroup);
+ this.serviceId = new ServiceId(this.serviceId.getName(), serviceGroup);
}
@JsonProperty("service_name")
public void setServiceName(String serviceName) {
- serviceId.setName(serviceName);
+ this.serviceId = new ServiceId(serviceName, this.serviceId.getServiceGroup());
}
public String getServiceName() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
index 5ef6517..90da776 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
@@ -18,26 +18,39 @@
*/
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toMap;
+
import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import javax.annotation.Nullable;
import org.apache.ambari.server.controller.internal.ProvisionAction;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
public class ProvisionClusterTemplate {
- private String blueprint;
+ private String blueprint = null;
+
@JsonProperty("default_password")
- private String defaultPassword;
+ private String defaultPassword = null;
+
@JsonProperty("config_recommendation_strategy")
private ConfigRecommendationStrategy configRecommendationStrategy;
+
@JsonProperty("provision_action")
private ProvisionAction provisionAction;
- private Collection<ProvisionClusterTemplate.Service> services;
- @JsonProperty("host_groups")
- private Collection<ProvisionClusterTemplate.HostGroup> hostGroups;
+
+ private Map<ServiceId, ProvisionClusterTemplate.Service> servicesById = Collections.emptyMap();
+
+ private Map<String, ProvisionClusterTemplate.HostGroup> hostGroups = Collections.emptyMap();
+
private Collection<Credential> credentials;
+
@JsonProperty("security")
private SecurityConfiguration securityConfiguration;
@@ -57,12 +70,21 @@ public class ProvisionClusterTemplate {
this.defaultPassword = defaultPassword;
}
+ public @Nullable Service getServiceById(ServiceId serviceId) {
+ return servicesById.get(serviceId);
+ }
+
+ @JsonProperty("services")
public Collection<Service> getServices() {
- return services;
+ return servicesById.values();
}
+ @JsonProperty("services")
public void setServices(Collection<Service> services) {
- this.services = services;
+ this.servicesById = services.stream().collect(toMap(
+ s -> s.getId(),
+ s -> s
+ ));
}
public Collection<Credential> getCredentials() {
@@ -97,21 +119,34 @@ public class ProvisionClusterTemplate {
this.provisionAction = provisionAction;
}
+ @JsonProperty("host_groups")
public Collection<HostGroup> getHostGroups() {
- return hostGroups;
+ return hostGroups.values();
+ }
+
+ public HostGroup getHostGroupByName(String name) {
+ return hostGroups.get(name);
}
+ @JsonProperty("host_groups")
public void setHostGroups(Collection<HostGroup> hostGroups) {
- this.hostGroups = hostGroups;
+ this.hostGroups = hostGroups.stream().collect(toMap(
+ hg -> hg.getName(),
+ hg -> hg
+ ));
+ }
+
+ public void validate() throws IllegalStateException {
+ getHostGroups().forEach(HostGroup::validate);
}
public static class HostGroup implements Configurable {
private String name;
@JsonIgnore
private Configuration configuration;
- private Collection<Host> hosts;
+ private Collection<Host> hosts = Collections.emptyList();
@JsonProperty("host_count")
- private Integer hostCount;
+ private int hostCount = 0;
@JsonProperty("host_predicate")
private String hostPredicate;
@@ -141,11 +176,11 @@ public class ProvisionClusterTemplate {
this.hosts = hosts;
}
- public Integer getHostCount() {
+ public int getHostCount() {
return hostCount;
}
- public void setHostCount(Integer hostCount) {
+ public void setHostCount(int hostCount) {
this.hostCount = hostCount;
}
@@ -156,6 +191,12 @@ public class ProvisionClusterTemplate {
public void setHostPredicate(String hostPredicate) {
this.hostPredicate = hostPredicate;
}
+
+ void validate() throws IllegalStateException {
+ Preconditions.checkState((hostCount == 0 && null == hostPredicate) || getHosts().isEmpty(),
+ "Invalid custer topology template. Host group %s must have either declatere its hosts or " +
+ "hostcount (and optionally host predicate)", name);
+ }
}
public static class Service implements Configurable {
@@ -175,6 +216,11 @@ public class ProvisionClusterTemplate {
this.configuration = configuration;
}
+ @JsonIgnore
+ public ServiceId getId() {
+ return new ServiceId(name, serviceGroup);
+ }
+
public String getName() {
return name;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
index c0c280f..cd99fa1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
@@ -51,16 +51,13 @@ public class ProvisionClusterTemplateFactory {
private void createObjectMapper() {
objectMapper = new ObjectMapper();
-// SimpleModule module = new SimpleModule("CustomModel", Version.unknownVersion());
-// SimpleAbstractTypeResolver resolver = new SimpleAbstractTypeResolver();
-// resolver.addMapping(HostGroupV2.class, HostGroupV2Impl.class);
-// module.setAbstractTypes(resolver);
-// objectMapper.registerModule(module);
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
}
public ProvisionClusterTemplate convertFromJson(String clusterTemplateJson) throws IOException {
- return objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class);
+ ProvisionClusterTemplate template = objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class);
+ template.validate();
+ return template;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
index 4a93ecd..a45386f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
@@ -36,7 +36,7 @@ public class Service implements Configurable {
private String type;
- private ServiceId id = new ServiceId();
+ private ServiceId id = new ServiceId(null, null);
private String stackId;
@@ -115,12 +115,12 @@ public class Service implements Configurable {
}
public void setName(String name) {
- this.id.setName(name);
+ this.id = new ServiceId(name, this.id.getServiceGroup());
}
public void setServiceGroup(ServiceGroup serviceGroup) {
this.serviceGroup = serviceGroup;
- this.id.setServiceGroup(serviceGroup.getName());
+ this.id = new ServiceId(this.id.getName(), serviceGroup.getName()) ;
}
@JsonProperty("stack_id")
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
index 2d81a07..9de0b3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
@@ -17,38 +17,33 @@
*/
package org.apache.ambari.server.topology;
+import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class ServiceId {
- private String serviceGroup;
- private String name;
+ private final String serviceGroup;
+ private final String name;
- public ServiceId() { }
+ @JsonCreator
+ public ServiceId(@JsonProperty("service_name") String name, @JsonProperty("service_group") String serviceGroup) {
+ this.name = name;
+ this.serviceGroup = serviceGroup;
+ }
public static ServiceId of(String name, String serviceGroup) {
- ServiceId id = new ServiceId();
- id.name = name;
- id.serviceGroup = serviceGroup;
- return id;
+ return new ServiceId(name, serviceGroup);
}
+ @JsonProperty("service_group")
public String getServiceGroup() {
return serviceGroup;
}
- @JsonProperty("service_group")
- public void setServiceGroup(String serviceGroup) {
- this.serviceGroup = serviceGroup;
- }
-
+ @JsonProperty("service_name")
public String getName() {
return name;
}
- @JsonProperty("service_name")
- public void setName(String name) {
- this.name = name;
- }
@Override
public boolean equals(Object o) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
index 78aa98c..ab9adea 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
@@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
public class BlueprintV2FactoryTest {
@@ -79,14 +80,11 @@ public class BlueprintV2FactoryTest {
assertEquals(2, bp.getServiceGroups().size());
Service zk1 = bp.getServiceGroup("CoreSG").getServiceByName("ZK1");
- Map<String, Map<String, String>> expectedProperties = new HashMap<>();
- Map<String, String> zooCfg = new HashMap<>();
- zooCfg.put("dataDir", "/zookeeper1");
- expectedProperties.put("zoo.cfg", zooCfg);
- Map<String, String> zookeeperEnv = new HashMap<>();
- zookeeperEnv.put("zk_user", "zkuser1");
- zookeeperEnv.put("zk_server_heapsize", "256MB");
- expectedProperties.put("zookeeper-env", zookeeperEnv);
+ Map<String, Map<String, String>> expectedProperties = ImmutableMap.of(
+ "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1"),
+ "zookeeper-env", ImmutableMap.of(
+ "zk_user", "zkuser1",
+ "zk_server_heapsize", "256MB"));
assertEquals(expectedProperties, zk1.getConfiguration().getProperties());
}
@@ -120,7 +118,7 @@ public class BlueprintV2FactoryTest {
verifyBlueprintStructure2(blueprintFactory.convertFromJson(BLUEPRINTV2_2_JSON));
}
- private void verifyBlueprintStructure2(BlueprintV2 bp) {
+ private void verifyBlueprintStructure2(BlueprintV2 bp) { // for "blueprintv2_2.json"
assertEquals(new StackId("HDP", "3.0.0"),
bp.getServiceGroups().iterator().next().getServices().iterator().next().getStack().getStackId());
assertEquals(1, bp.getStackIds().size());
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
index b02a00a..3bb6ea4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
@@ -18,23 +18,68 @@
*/
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
+import java.util.Map;
import org.junit.Test;
import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
import com.google.common.io.Resources;
public class ProvisionClusterTemplateTest {
- public static final String CLUSTER_TEMPLATE = getResource("blueprintv2/cluster_template_v2.json");
+ public static final String CLUSTER_TEMPLATE =
+ getResource("blueprintv2/cluster_template_v2.json");
+ public static final String CLUSTER_TEMPLATE_INVALID =
+ getResource("blueprintv2/cluster_template_v2_invalid_hostgroup.json");
@Test
public void testProvisionClusterTemplate() throws Exception {
ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory();
ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE);
- System.out.println(template);
+ verifyClusterTemplate(template);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testProvisionClusterTemplateInvalidTemplate() throws Exception {
+ ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory();
+ ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE_INVALID);
+ }
+
+
+ private void verifyClusterTemplate(ProvisionClusterTemplate template) {
+ ProvisionClusterTemplate.Service zk1 = template.getServiceById(ServiceId.of("ZK1", "CORE_SG"));
+ assertNotNull(zk1);
+ Map<String, Map<String, String>> expectedZkProperties = ImmutableMap.of(
+ "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1"));
+ assertEquals(expectedZkProperties, zk1.getConfiguration().getProperties());
+
+ ProvisionClusterTemplate.Service hdfs = template.getServiceById(ServiceId.of("HDFS", "CORE_SG"));
+ Map<String, Map<String, String>> expectedHdfsProperties = ImmutableMap.of(
+ "hdfs-site", ImmutableMap.of("property-name", "property-value"));
+ assertNotNull(hdfs);
+ assertEquals(expectedHdfsProperties, hdfs.getConfiguration().getProperties());
+
+ ProvisionClusterTemplate.HostGroup hostGroup1 = template.getHostGroupByName("host-group-1");
+ assertNotNull(hostGroup1);
+ assertEquals(2, hostGroup1.getHosts().size());
+ assertEquals(0, hostGroup1.getHostCount());
+ assertEquals(ImmutableSet.of("host.domain.com", "host2.domain.com"),
+ hostGroup1.getHosts().stream().map(host -> host.getFqdn()).collect(toSet()));
+ hostGroup1.getHosts().forEach(host -> assertEquals("/dc1/rack1", host.getRackInfo()));
+
+ ProvisionClusterTemplate.HostGroup hostGroup2 = template.getHostGroupByName("host-group-2");
+ assertNotNull(hostGroup2);
+ assertEquals(0, hostGroup2.getHosts().size());
+ assertEquals(2, hostGroup2.getHostCount());
+ assertEquals("Hosts/os_type=centos6&Hosts/cpu_count=2", hostGroup2.getHostPredicate());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
index 1fb0b04..d80f2f5 100644
--- a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
+++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
@@ -9,7 +9,7 @@
{
"zoo.cfg": {
"properties": {
- "dataDir": "/zookeeper2"
+ "dataDir": "/zookeeper1"
}
}
}
@@ -44,13 +44,22 @@
],
"hosts": [
{
- "fqdn": "host.domain.com"
+ "fqdn": "host.domain.com",
+ "rack_info": "/dc1/rack1"
},
{
- "fqdn": "host2.domain.com"
+ "fqdn": "host2.domain.com",
+ "rack_info": "/dc1/rack1"
}
]
+ },
+ {
+ "name": "host-group-2",
+ "configurations": [],
+ "host_count": "2",
+ "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2"
}
+
],
"credentials": [
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
new file mode 100644
index 0000000..abe0aaf
--- /dev/null
+++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
@@ -0,0 +1,70 @@
+{
+ "blueprint": "blueprint-name",
+ "default_password": "super-secret-password",
+ "services": [
+ {
+ "service_group": "CORE_SG",
+ "name": "ZK1",
+ "configurations": [
+ {
+ "zoo.cfg": {
+ "properties": {
+ "dataDir": "/zookeeper1"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "service_group": "CORE_SG",
+ "name": "HDFS",
+ "configurations": [
+ {
+ "hdfs-site": {
+ "properties": {
+ "property-name": "property-value"
+ }
+ }
+ }
+ ]
+ }
+
+ ],
+ "host_groups": [
+ {
+ "name": "host-group-1",
+ "configurations": [
+ {
+ "yarn-site": {
+ "properties": {
+ "property-name": "property-value"
+ }
+ }
+ }
+ ],
+ "hosts": [
+ {
+ "fqdn": "host.domain.com",
+ "rack_info": "/dc1/rack1"
+ },
+ {
+ "fqdn": "host2.domain.com",
+ "rack_info": "/dc1/rack1"
+ }
+ ],
+ "host_count": "2",
+ "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2"
+ }
+ ],
+ "credentials": [
+ {
+ "alias": "kdc.admin.credential",
+ "principal": "principal",
+ "key": "key",
+ "type": "TEMPORARY"
+ }
+ ],
+ "security": {
+ "type": "NONE"
+ }
+}