You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ad...@apache.org on 2018/07/02 16:28:11 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: AMBARI-24220.
Override duplicate mpacks with the one from cluster template (benyoka)
(#1654)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 4988a2f AMBARI-24220. Override duplicate mpacks with the one from cluster template (benyoka) (#1654)
4988a2f is described below
commit 4988a2fffda4655ff8a5fffadf738c905065b409
Author: benyoka <be...@users.noreply.github.com>
AuthorDate: Mon Jul 2 18:28:07 2018 +0200
AMBARI-24220. Override duplicate mpacks with the one from cluster template (benyoka) (#1654)
---
.../ClusterTemplateArtifactPasswordReplacer.java | 2 +-
.../BlueprintBasedClusterProvisionRequest.java | 24 +++++--
.../ambari/server/topology/MpackInstance.java | 52 +++++++++++++-
.../BlueprintBasedClusterProvisionRequestTest.java | 84 ++++++++++++++++++++++
4 files changed, 155 insertions(+), 7 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterTemplateArtifactPasswordReplacer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterTemplateArtifactPasswordReplacer.java
index 3caeb0f..f3c785e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterTemplateArtifactPasswordReplacer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterTemplateArtifactPasswordReplacer.java
@@ -152,7 +152,7 @@ public class ClusterTemplateArtifactPasswordReplacer {
List<Map<String, Object>> mpackInstances =
(List<Map<String, Object>>)artifactData.getOrDefault("mpack_instances", emptyList());
return mpackInstances.stream().
- map(mpackMap -> new StackId((String)mpackMap.get("name"), (String)mpackMap.get("version"))).
+ map(mpackMap -> new StackId((String)mpackMap.get("type"), (String)mpackMap.get("version"))).
collect(toList());
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java
index ed2ab4b..e79321b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -52,9 +53,8 @@ public class BlueprintBasedClusterProvisionRequest implements Blueprint, Provisi
this.blueprint = blueprint;
this.request = request;
stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), request.getStackIds()));
- mpacks = ImmutableSet.<MpackInstance>builder().
- addAll(blueprint.getMpacks()).
- addAll(request.getMpacks()).build();
+
+ mpacks = mergeMpacks(blueprint.getMpacks(), request.getMpacks());
stack = ambariContext.composeStacks(stackIds);
@@ -65,6 +65,23 @@ public class BlueprintBasedClusterProvisionRequest implements Blueprint, Provisi
}
}
+ /**
+ * Override mpacks from blueprint with mpacks from provision cluster request that have the same type, name
+ * and version (this triplet needs to be unique)
+ */
+ Set<MpackInstance> mergeMpacks(Collection<MpackInstance> blueprintMpacks, Collection<MpackInstance> clusterTemplateMpacks) {
+ Map<MpackInstance.Key, MpackInstance> mpackInstanceMap = new HashMap<>();
+ blueprintMpacks.forEach(mpack -> mpackInstanceMap.put(mpack.getKey(), mpack));
+ clusterTemplateMpacks.forEach( mpack -> {
+ MpackInstance.Key key = mpack.getKey();
+ if (mpackInstanceMap.containsKey(key)) {
+ LOG.info("Overriding mpack from blueprint with mpack from provision cluster request: {}", key);
+ }
+ mpackInstanceMap.put(key, mpack);
+ });
+ return ImmutableSet.copyOf(mpackInstanceMap.values());
+ }
+
@Override
public String getName() {
return blueprint.getName();
@@ -193,5 +210,4 @@ public class BlueprintBasedClusterProvisionRequest implements Blueprint, Provisi
public Set<MpackInstance> getAllMpacks() {
return mpacks;
}
-
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java
index 5756941..3497ee5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.topology;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Objects;
import org.apache.ambari.server.orm.entities.BlueprintEntity;
import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity;
@@ -33,6 +34,7 @@ import org.apache.ambari.server.state.StackId;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects;
public class MpackInstance implements Configurable {
@JsonProperty("name")
@@ -144,6 +146,53 @@ public class MpackInstance implements Configurable {
return mpackInstanceEntity;
}
+ public Key getKey() {
+ return new Key(this);
+ }
+
+ /**
+ * Represents the 'business key' of (all properties that uniquely identifies) an mpack instance.
+ */
+ public static class Key {
+ public final String name;
+ public final String type;
+ public final String version;
+
+ Key(MpackInstance instance) {
+ this(instance.getMpackName(), instance.getMpackType(), instance.getMpackVersion());
+ }
+
+ Key(String name, String type, String version) {
+ this.name = name;
+ this.type = type;
+ this.version = version;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Key key = (Key) o;
+ return Objects.equals(name, key.name) &&
+ Objects.equals(type, key.type) &&
+ Objects.equals(version, key.version);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, type, version);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("name", name)
+ .add("type", type)
+ .add("version", version)
+ .toString();
+ }
+ }
+
/**
* Used during conversion to {@link MpackInstanceEntity}. Sets the common properties that are shared across all
* {@link MpackInstanceEntity} subclasses. Bidirectional relations are handled for {@link MpackInstanceConfigEntity},
@@ -175,7 +224,7 @@ public class MpackInstance implements Configurable {
public static MpackInstance fromEntity(MpackInstanceEntity entity) {
MpackInstance mpack = new MpackInstance(entity.getMpackName(), entity.getMpackType(), entity.getMpackVersion(), entity.getMpackUri(), BlueprintImpl.fromConfigEntities(entity.getConfigurations()));
- for (MpackInstanceServiceEntity serviceEntity: entity.getServiceInstances()) {
+ for (MpackInstanceServiceEntity serviceEntity : entity.getServiceInstances()) {
ServiceInstance serviceInstance = new ServiceInstance();
serviceInstance.setName(serviceEntity.getName());
serviceInstance.setType(serviceEntity.getType());
@@ -186,5 +235,4 @@ public class MpackInstance implements Configurable {
return mpack;
}
-
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequestTest.java
index e0f6549..9e8af4d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequestTest.java
@@ -17,11 +17,18 @@
*/
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
+import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import java.util.Collection;
+import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
import org.apache.ambari.server.controller.internal.StackDefinition;
@@ -36,6 +43,16 @@ public class BlueprintBasedClusterProvisionRequestTest {
private static final StackId STACK_ID = new StackId("HDP-2.6");
private static final Set<StackId> STACK_IDS = ImmutableSet.of(STACK_ID);
+ private static final String HDPCORE = "HDPCORE";
+ private static final String EDW = "EDW";
+ private static final String ODS = "ODS";
+ private static final String ODS_MARKETING = "ODS MARKETING";
+ private static final String ODS_RND = "ODS R&D";
+ private static final String V10 = "1.0.0.0";
+ private static final String V11 = "1.1.0.0";
+ private static final String URI_FROM_BLUEPRINT = "http://from.blueprint";
+ private static final String URI_FROM_PROVISION_REQUEST = "http://from.provision.request";
+
@Test(expected = IllegalArgumentException.class) // THEN
public void clusterCannotRelaxBlueprintSecurity() {
// GIVEN
@@ -52,6 +69,70 @@ public class BlueprintBasedClusterProvisionRequestTest {
new BlueprintBasedClusterProvisionRequest(context, null, blueprint, request);
}
+ @Test
+ public void mergeMpacks() throws Exception {
+ // GIVEN
+ Collection<MpackInstance> blueprintMpacks = ImmutableSet.of(
+ mpack(HDPCORE, HDPCORE, V10, URI_FROM_BLUEPRINT), // this will be overridden
+ mpack(ODS, ODS_MARKETING, V10, URI_FROM_BLUEPRINT),
+ mpack(EDW, EDW, V10, URI_FROM_BLUEPRINT));
+ Collection<MpackInstance> clusterTemplateMpacks = ImmutableSet.of(
+ mpack(HDPCORE, HDPCORE, V10, URI_FROM_PROVISION_REQUEST),
+ mpack(ODS, ODS_RND, V10, URI_FROM_PROVISION_REQUEST),
+ mpack(EDW, EDW, V11, URI_FROM_PROVISION_REQUEST));
+
+ // WHEN
+ BlueprintBasedClusterProvisionRequest request = createRequest(blueprintMpacks, clusterTemplateMpacks);
+
+ // THEN
+ assertEquals(5, request.getAllMpacks().size()); // one less than bp + cluster req combined due to one override
+ Map<MpackInstance.Key, MpackInstance> mpacks =
+ request.getAllMpacks().stream().collect(toMap(MpackInstance::getKey, Function.identity()));
+ assertEquals(
+ URI_FROM_BLUEPRINT,
+ mpacks.get(new MpackInstance.Key(ODS, ODS_MARKETING, V10)).getUrl());
+ assertEquals(
+ URI_FROM_BLUEPRINT,
+ mpacks.get(new MpackInstance.Key(EDW, EDW, V10)).getUrl());
+ assertEquals(
+ "mpack definition in blueprint was not properly overriden from cluster template",
+ URI_FROM_PROVISION_REQUEST,
+ mpacks.get(new MpackInstance.Key(HDPCORE, HDPCORE, V10)).getUrl());
+ assertEquals(
+ URI_FROM_PROVISION_REQUEST,
+ mpacks.get(new MpackInstance.Key(ODS, ODS_RND, V10)).getUrl());
+ assertEquals(
+ URI_FROM_PROVISION_REQUEST,
+ mpacks.get(new MpackInstance.Key(EDW, EDW, V11)).getUrl());
+ }
+
+ private BlueprintBasedClusterProvisionRequest createRequest(Collection<MpackInstance> blueprintMpacks,
+ Collection<MpackInstance> clusterTemplateMpacks) {
+ Blueprint blueprint = createNiceMock(Blueprint.class);
+ SecurityConfiguration secure = new SecurityConfiguration(SecurityType.NONE);
+ expect(blueprint.getSecurity()).andReturn(secure).anyTimes();
+ expect(blueprint.getStackIds()).andReturn(
+ blueprintMpacks.stream()
+ .map( MpackInstance::getStackId )
+ .collect(toSet()))
+ .anyTimes();
+ expect(blueprint.getMpacks()).andReturn(blueprintMpacks).anyTimes();
+
+ ProvisionClusterRequest request = createNiceMock(ProvisionClusterRequest.class);
+ expect(request.getSecurityConfiguration()).andReturn(SecurityConfiguration.NONE).anyTimes();
+ expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes();
+ expect(request.getMpacks()).andReturn(clusterTemplateMpacks).anyTimes();
+
+ AmbariContext context = createNiceMock(AmbariContext.class);
+ StackDefinition stack = createNiceMock(StackDefinition.class);
+ expect(context.composeStacks(anyObject())).andReturn(stack).anyTimes();
+
+ replay(context, stack, blueprint, request);
+
+ return new BlueprintBasedClusterProvisionRequest(context, null, blueprint, request);
+ }
+
+
private ProvisionClusterRequest insecureCluster() {
ProvisionClusterRequest request = createNiceMock(ProvisionClusterRequest.class);
expect(request.getSecurityConfiguration()).andReturn(SecurityConfiguration.NONE).anyTimes();
@@ -69,4 +150,7 @@ public class BlueprintBasedClusterProvisionRequestTest {
return blueprint;
}
+ private static final MpackInstance mpack(String name, String type, String version, String uri) {
+ return new MpackInstance(name, type, version, uri, null);
+ }
}