You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2017/10/16 18:15:01 UTC
[1/4] ambari git commit: use V2 objects in TopologyManager
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 b0ff5da46 -> 65d44cd50
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
index f50e60f..8feb979 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
@@ -19,36 +19,34 @@
package org.apache.ambari.server.topology;
-import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START;
-import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.RequestStatusResponse;
+import org.apache.ambari.server.controller.internal.ConfigurationContext;
import org.apache.ambari.server.controller.internal.ProvisionAction;
import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+
+import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START;
+import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY;
+
/**
* Represents a cluster topology.
* Topology includes the the associated blueprint, cluster configuration and hostgroup -> host mapping.
*/
public class ClusterTopologyImpl implements ClusterTopology {
+ private final static Logger LOG = LoggerFactory.getLogger(ClusterTopologyImpl.class);
+
private Long clusterId;
//todo: currently topology is only associated with a single bp
//todo: this will need to change to allow usage of multiple bp's for the same cluster
//todo: for example: provision using bp1 and scale using bp2
- private Blueprint blueprint;
- private Configuration configuration;
+ private BlueprintV2 blueprint;
+ private Collection<Service> serviceConfigs;
private ConfigRecommendationStrategy configRecommendationStrategy;
private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START;
private Map<String, AdvisedConfiguration> advisedConfigurations = new HashMap<>();
@@ -56,16 +54,13 @@ public class ClusterTopologyImpl implements ClusterTopology {
private final AmbariContext ambariContext;
private final String defaultPassword;
- private final static Logger LOG = LoggerFactory.getLogger(ClusterTopologyImpl.class);
-
-
//todo: will need to convert all usages of hostgroup name to use fully qualified name (BP/HG)
//todo: for now, restrict scaling to the same BP
public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topologyRequest) throws InvalidTopologyException {
this.clusterId = topologyRequest.getClusterId();
// provision cluster currently requires that all hostgroups have same BP so it is ok to use root level BP here
this.blueprint = topologyRequest.getBlueprint();
- this.configuration = topologyRequest.getConfiguration();
+ this.serviceConfigs = topologyRequest.getServiceConfigs();
if (topologyRequest instanceof ProvisionClusterRequest) {
this.defaultPassword = ((ProvisionClusterRequest) topologyRequest).getDefaultPassword();
} else {
@@ -95,13 +90,18 @@ public class ClusterTopologyImpl implements ClusterTopology {
}
@Override
- public Blueprint getBlueprint() {
+ public BlueprintV2 getBlueprint() {
return blueprint;
}
@Override
+ @Deprecated
public Configuration getConfiguration() {
- return configuration;
+ return null;
+ }
+
+ public Collection<Service> getServiceConfigs() {
+ return serviceConfigs;
}
@Override
@@ -113,7 +113,7 @@ public class ClusterTopologyImpl implements ClusterTopology {
@Override
public Collection<String> getHostGroupsForComponent(String component) {
Collection<String> resultGroups = new ArrayList<>();
- for (HostGroup group : getBlueprint().getHostGroups().values() ) {
+ for (HostGroupV2 group : getBlueprint().getHostGroups().values() ) {
if (group.getComponentNames().contains(component)) {
resultGroups.add(group.getName());
}
@@ -178,49 +178,48 @@ public class ClusterTopologyImpl implements ClusterTopology {
}
@Override
- public boolean isNameNodeHAEnabled() {
- return isNameNodeHAEnabled(configuration.getFullProperties());
- }
-
- public static boolean isNameNodeHAEnabled(Map<String, Map<String, String>> configurationProperties) {
- return configurationProperties.containsKey("hdfs-site") &&
- (configurationProperties.get("hdfs-site").containsKey("dfs.nameservices") ||
- configurationProperties.get("hdfs-site").containsKey("dfs.internal.nameservices"));
- }
-
- @Override
- public boolean isYarnResourceManagerHAEnabled() {
- return isYarnResourceManagerHAEnabled(configuration.getFullProperties());
+ public boolean isNameNodeHAEnabled(ConfigurationContext configurationContext) {
+ return configurationContext.isNameNodeHAEnabled();
}
/**
* Static convenience function to determine if Yarn ResourceManager HA is enabled
- * @param configProperties configuration properties for this cluster
+ * @param configurationContext configuration context
* @return true if Yarn ResourceManager HA is enabled
* false if Yarn ResourceManager HA is not enabled
*/
- static boolean isYarnResourceManagerHAEnabled(Map<String, Map<String, String>> configProperties) {
- return configProperties.containsKey("yarn-site") && configProperties.get("yarn-site").containsKey("yarn.resourcemanager.ha.enabled")
- && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true");
+ @Override
+ public boolean isYarnResourceManagerHAEnabled(ConfigurationContext configurationContext) {
+ return configurationContext.isYarnResourceManagerHAEnabled();
}
private void validateTopology()
throws InvalidTopologyException {
- if(isNameNodeHAEnabled()){
+ Collection<Service> hdfsServices = getBlueprint().getServicesByType("HDFS");
+ for (Service hdfsService : hdfsServices) {
+ ConfigurationContext configContext = new ConfigurationContext(hdfsService.getStack(), hdfsService.getConfiguration());
+ if(isNameNodeHAEnabled(configContext)) {
+
Collection<String> nnHosts = getHostAssignmentsForComponent("NAMENODE");
if (nnHosts.size() != 2) {
- throw new InvalidTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " +
- nnHosts.size() + " Hosts: " + nnHosts);
+ throw new InvalidTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " +
+ nnHosts.size() + " Hosts: " + nnHosts);
}
- Map<String, String> hadoopEnvConfig = configuration.getFullProperties().get("hadoop-env");
+
+ Map<String, String> hadoopEnvConfig = hdfsService.getConfiguration().getProperties().get("hadoop-env");
if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) {
- if((!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")))
- || (!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")))){
- throw new IllegalArgumentException("NAMENODE HA hosts mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected hosts are: " + nnHosts);
- }
+ if((!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")))
+ || (!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")))){
+ throw new IllegalArgumentException("NAMENODE HA hosts mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected hosts are: " + nnHosts);
+ }
}
+
+ }
+
}
+
+
}
@Override
@@ -232,7 +231,7 @@ public class ClusterTopologyImpl implements ClusterTopology {
public RequestStatusResponse installHost(String hostName, boolean skipInstallTaskCreate, boolean skipFailure) {
try {
String hostGroupName = getHostGroupForHost(hostName);
- HostGroup hostGroup = this.blueprint.getHostGroup(hostGroupName);
+ HostGroupV2 hostGroup = this.blueprint.getHostGroup(hostGroupName);
Collection<String> skipInstallForComponents = new ArrayList<>();
if (skipInstallTaskCreate) {
@@ -257,7 +256,7 @@ public class ClusterTopologyImpl implements ClusterTopology {
public RequestStatusResponse startHost(String hostName, boolean skipFailure) {
try {
String hostGroupName = getHostGroupForHost(hostName);
- HostGroup hostGroup = this.blueprint.getHostGroup(hostGroupName);
+ HostGroupV2 hostGroup = this.blueprint.getHostGroup(hostGroupName);
// get the set of components that are marked as INSTALL_ONLY
// for this hostgroup
@@ -321,7 +320,7 @@ public class ClusterTopologyImpl implements ClusterTopology {
String hostGroupName = requestedHostGroupInfo.getHostGroupName();
//todo: doesn't support using a different blueprint for update (scaling)
- HostGroup baseHostGroup = getBlueprint().getHostGroup(hostGroupName);
+ HostGroupV2 baseHostGroup = getBlueprint().getHostGroup(hostGroupName);
if (baseHostGroup == null) {
throw new IllegalArgumentException("Invalid host_group specified: " + hostGroupName +
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
index 175fe99..d7c08f4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
@@ -23,6 +23,8 @@ import org.apache.ambari.server.controller.internal.ProvisionAction;
public class ComponentV2 {
+ private final String type;
+
private final String name;
private final Service service;
@@ -31,12 +33,17 @@ public class ComponentV2 {
private final Configuration configuration;
+ public ComponentV2(String type, Service service) {
+ this(type, type, service, null, null);
+ }
+
- public ComponentV2(String name, Service service) {
- this(name, service, null, null);
+ public ComponentV2(String type, String name, Service service) {
+ this(type, name, service, null, null);
}
- public ComponentV2(String name, Service service, ProvisionAction provisionAction, Configuration configuration) {
+ public ComponentV2(String type, String name, Service service, ProvisionAction provisionAction, Configuration configuration) {
+ this.type = type;
this.name = name;
this.service = service;
this.provisionAction = provisionAction;
@@ -52,6 +59,10 @@ public class ComponentV2 {
return this.name;
}
+ public String getType() {
+ return type;
+ }
+
/**
* Gets the provision action associated with this component.
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
index 4648412..4db3a92 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
@@ -18,17 +18,17 @@
package org.apache.ambari.server.topology;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
import org.apache.ambari.server.api.predicate.InvalidQueryException;
import org.apache.ambari.server.api.predicate.PredicateCompiler;
import org.apache.ambari.server.controller.spi.Predicate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
/**
* Host Group information specific to a cluster instance.
*/
@@ -57,14 +57,14 @@ public class HostGroupInfo {
private final Map<String, String> hostRackInfo = new HashMap<>();
/**
- * explicitly specified host count
+ * List of services
*/
- private int requested_count = 0;
+ protected Collection<Service> serviceConfigs;
/**
- * host group scoped configuration
+ * explicitly specified host count
*/
- Configuration configuration;
+ private int requested_count = 0;
/**
* explicitly specified host predicate string
@@ -110,6 +110,10 @@ public class HostGroupInfo {
}
}
+ public Collection<Service> getServiceConfigs() {
+ return serviceConfigs;
+ }
+
/**
* Get the requested host count.
* This is either the user specified value or
@@ -166,7 +170,7 @@ public class HostGroupInfo {
* @param configuration configuration instance
*/
public void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
+
}
/**
@@ -175,8 +179,9 @@ public class HostGroupInfo {
* @return associated host group scoped configuration or null if no configuration
* is specified for the host group
*/
+ @Deprecated
public Configuration getConfiguration() {
- return configuration;
+ return null;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupV2.java
index df26b68..fd0f966 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupV2.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupV2.java
@@ -110,9 +110,15 @@ public interface HostGroupV2 {
*
* @return host group configuration
*/
+ @Deprecated
Configuration getConfiguration();
/**
+ * List of services
+ */
+ Collection<Service> getServiceConfigs();
+
+ /**
* Get the cardinality value that was specified for the host group.
* This is simply meta-data for the stack that a deployer can use
* and this information is not used by ambari.
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
index 7045912..d8390c6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java
@@ -18,22 +18,12 @@
package org.apache.ambari.server.topology;
-import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START;
-import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY;
-import static org.apache.ambari.server.controller.internal.ProvisionAction.START_ONLY;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.api.predicate.InvalidQueryException;
import org.apache.ambari.server.api.predicate.PredicateCompiler;
import org.apache.ambari.server.controller.internal.HostResourceProvider;
import org.apache.ambari.server.controller.internal.ResourceImpl;
-import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.internal.StackV2;
import org.apache.ambari.server.controller.spi.Predicate;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
@@ -41,14 +31,14 @@ import org.apache.ambari.server.orm.entities.TopologyHostRequestEntity;
import org.apache.ambari.server.orm.entities.TopologyHostTaskEntity;
import org.apache.ambari.server.orm.entities.TopologyLogicalTaskEntity;
import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.topology.tasks.InstallHostTask;
-import org.apache.ambari.server.topology.tasks.PersistHostResourcesTask;
-import org.apache.ambari.server.topology.tasks.RegisterWithConfigGroupTask;
-import org.apache.ambari.server.topology.tasks.StartHostTask;
-import org.apache.ambari.server.topology.tasks.TopologyTask;
+import org.apache.ambari.server.topology.tasks.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+
+import static org.apache.ambari.server.controller.internal.ProvisionAction.*;
+
/**
@@ -60,7 +50,7 @@ public class HostRequest implements Comparable<HostRequest> {
private long requestId;
private String blueprint;
- private HostGroup hostGroup;
+ private HostGroupV2 hostGroup;
private String hostgroupName;
private Predicate predicate;
private String hostname = null;
@@ -84,7 +74,7 @@ public class HostRequest implements Comparable<HostRequest> {
private static PredicateCompiler predicateCompiler = new PredicateCompiler();
public HostRequest(long requestId, long id, long clusterId, String hostname, String blueprintName,
- HostGroup hostGroup, Predicate predicate, ClusterTopology topology, boolean skipFailure) {
+ HostGroupV2 hostGroup, Predicate predicate, ClusterTopology topology, boolean skipFailure) {
this.requestId = requestId;
this.id = id;
this.clusterId = clusterId;
@@ -164,7 +154,7 @@ public class HostRequest implements Comparable<HostRequest> {
return blueprint;
}
- public HostGroup getHostGroup() {
+ public HostGroupV2 getHostGroup() {
return hostGroup;
}
@@ -206,13 +196,13 @@ public class HostRequest implements Comparable<HostRequest> {
}
// lower level logical component level tasks which get mapped to physical tasks
- HostGroup hostGroup = getHostGroup();
+ HostGroupV2 hostGroup = getHostGroup();
Collection<String> startOnlyComponents = hostGroup.getComponentNames(START_ONLY);
Collection<String> installOnlyComponents = hostGroup.getComponentNames(INSTALL_ONLY);
Collection<String> installAndStartComponents = hostGroup.getComponentNames(INSTALL_AND_START);
- for (String component : hostGroup.getComponentNames()) {
- if (component == null || component.equals("AMBARI_SERVER")) {
+ for (ComponentV2 component : hostGroup.getComponents()) {
+ if (component == null || component.getType().equals("AMBARI_SERVER")) {
LOG.info("Skipping component {} when creating request\n", component);
continue;
}
@@ -222,31 +212,31 @@ public class HostRequest implements Comparable<HostRequest> {
"PENDING HOST ASSIGNMENT : HOSTGROUP=" + getHostgroupName();
AmbariContext context = topology.getAmbariContext();
- Stack stack = hostGroup.getStack();
+ StackV2 stack = component.getService().getStack();
// Skip INSTALL task in case server component is marked as START_ONLY, or the cluster provision_action is
// START_ONLY, unless component is marked with INSTALL_ONLY or INSTALL_AND_START.
- if (startOnlyComponents.contains(component) || (skipInstallTaskCreate &&
- !installOnlyComponents.contains(component) && !installAndStartComponents.contains(component))
- && stack != null && !stack.getComponentInfo(component).isClient()) {
- LOG.info("Skipping create of INSTALL task for {} on {}.", component, hostName);
+ if (startOnlyComponents.contains(component.getName()) || (skipInstallTaskCreate &&
+ !installOnlyComponents.contains(component.getName()) && !installAndStartComponents.contains(component.getName()))
+ && stack != null && !stack.getComponentInfo(component.getType()).isClient()) {
+ LOG.info("Skipping create of INSTALL task for {} on {}.", component.getName(), hostName);
} else {
HostRoleCommand logicalInstallTask = context.createAmbariTask(
- getRequestId(), id, component, hostName, AmbariContext.TaskType.INSTALL, skipFailure);
+ getRequestId(), id, component.getName(), hostName, AmbariContext.TaskType.INSTALL, skipFailure);
logicalTasks.put(logicalInstallTask.getTaskId(), logicalInstallTask);
- logicalTaskMap.get(installTask).put(component, logicalInstallTask.getTaskId());
+ logicalTaskMap.get(installTask).put(component.getName(), logicalInstallTask.getTaskId());
}
// Skip START task if component is a client, or ir marked as INSTALL_ONLY or cluster provision_action is
// INSTALL_ONLY
- if (installOnlyComponents.contains(component) || skipStartTaskCreate ||
- (stack != null && stack.getComponentInfo(component).isClient())) {
- LOG.info("Skipping create of START task for {} on {}.", component, hostName);
+ if (installOnlyComponents.contains(component.getName()) || skipStartTaskCreate ||
+ (stack != null && stack.getComponentInfo(component.getType()).isClient())) {
+ LOG.info("Skipping create of START task for {} on {}.", component.getName(), hostName);
} else {
HostRoleCommand logicalStartTask = context.createAmbariTask(
- getRequestId(), id, component, hostName, AmbariContext.TaskType.START, skipFailure);
+ getRequestId(), id, component.getName(), hostName, AmbariContext.TaskType.START, skipFailure);
logicalTasks.put(logicalStartTask.getTaskId(), logicalStartTask);
- logicalTaskMap.get(startTask).put(component, logicalStartTask.getTaskId());
+ logicalTaskMap.get(startTask).put(component.getName(), logicalStartTask.getTaskId());
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
index b63bbad..1049b90 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
@@ -18,18 +18,7 @@
package org.apache.ambari.server.topology;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicLong;
-
+import com.google.common.collect.Iterables;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
@@ -39,17 +28,14 @@ import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.ShortTaskStatus;
import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
-import org.apache.ambari.server.orm.entities.StageEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostGroupEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostInfoEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostRequestEntity;
-import org.apache.ambari.server.orm.entities.TopologyLogicalRequestEntity;
+import org.apache.ambari.server.orm.entities.*;
import org.apache.ambari.server.state.Host;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Iterables;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
/**
@@ -222,7 +208,7 @@ public class LogicalRequest extends Request {
//todo: synchronization
for (HostRequest hostRequest : allHostRequests) {
- HostGroup hostGroup = hostRequest.getHostGroup();
+ HostGroupV2 hostGroup = hostRequest.getHostGroup();
for (String host : topology.getHostGroupInfo().get(hostGroup.getName()).getHostNames()) {
Collection<String> hostComponents = hostComponentMap.get(host);
if (hostComponents == null) {
@@ -407,7 +393,7 @@ public class LogicalRequest extends Request {
private void createHostRequests(TopologyRequest request, ClusterTopology topology) {
Map<String, HostGroupInfo> hostGroupInfoMap = request.getHostGroupInfo();
- Blueprint blueprint = topology.getBlueprint();
+ BlueprintV2 blueprint = topology.getBlueprint();
boolean skipFailure = topology.getBlueprint().shouldSkipFailure();
for (HostGroupInfo hostGroupInfo : hostGroupInfoMap.values()) {
String groupName = hostGroupInfo.getHostGroupName();
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java
index 12af131..dfeb99b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java
@@ -18,43 +18,23 @@
package org.apache.ambari.server.topology;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.inject.Singleton;
-
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.persist.Transactional;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.api.predicate.InvalidQueryException;
import org.apache.ambari.server.controller.internal.BaseClusterRequest;
-import org.apache.ambari.server.orm.dao.HostDAO;
-import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
-import org.apache.ambari.server.orm.dao.TopologyHostGroupDAO;
-import org.apache.ambari.server.orm.dao.TopologyHostInfoDAO;
-import org.apache.ambari.server.orm.dao.TopologyHostRequestDAO;
-import org.apache.ambari.server.orm.dao.TopologyLogicalRequestDAO;
-import org.apache.ambari.server.orm.dao.TopologyLogicalTaskDAO;
-import org.apache.ambari.server.orm.dao.TopologyRequestDAO;
-import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostGroupEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostInfoEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostRequestEntity;
-import org.apache.ambari.server.orm.entities.TopologyHostTaskEntity;
-import org.apache.ambari.server.orm.entities.TopologyLogicalRequestEntity;
-import org.apache.ambari.server.orm.entities.TopologyLogicalTaskEntity;
-import org.apache.ambari.server.orm.entities.TopologyRequestEntity;
+import org.apache.ambari.server.orm.dao.*;
+import org.apache.ambari.server.orm.entities.*;
import org.apache.ambari.server.stack.NoSuchStackException;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.topology.tasks.TopologyTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.persist.Transactional;
+import javax.inject.Singleton;
+import java.util.*;
/**
* Implementation which uses Ambari Database DAO and Entity objects for persistence
@@ -378,8 +358,8 @@ public class PersistedStateImpl implements PersistedState {
private final Long clusterId;
private final Type type;
private final String description;
- private final Blueprint blueprint;
- private final Configuration configuration;
+ private final BlueprintV2 blueprint;
+ private final Collection<Service> services;
private final Map<String, HostGroupInfo> hostGroupInfoMap = new HashMap<>();
public ReplayedTopologyRequest(TopologyRequestEntity entity, BlueprintFactory blueprintFactory) {
@@ -392,8 +372,11 @@ public class PersistedStateImpl implements PersistedState {
} catch (NoSuchStackException e) {
throw new RuntimeException("Unable to load blueprint while replaying topology request: " + e, e);
}
- configuration = createConfiguration(entity.getClusterProperties(), entity.getClusterAttributes());
- configuration.setParentConfiguration(blueprint.getConfiguration());
+ //TODO load services, merge servie configs from Cluster template with service configs from Blueprint
+ services = new ArrayList<>();
+ //configuration = createConfiguration(entity.getClusterProperties(), entity.getClusterAttributes());
+ //configuration.setParentConfiguration(blueprint.getConfiguration());
+
parseHostGroupInfo(entity);
}
@@ -409,13 +392,19 @@ public class PersistedStateImpl implements PersistedState {
}
@Override
- public Blueprint getBlueprint() {
+ public BlueprintV2 getBlueprint() {
return blueprint;
}
@Override
+ @Deprecated
public Configuration getConfiguration() {
- return configuration;
+ return null;
+ }
+
+ @Override
+ public Collection<Service> getServiceConfigs() {
+ return services;
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
index 62acdfd..f897f7b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
@@ -19,9 +19,9 @@
package org.apache.ambari.server.topology;
-import java.util.Set;
+import org.apache.ambari.server.controller.internal.StackV2;
-import org.apache.ambari.server.controller.internal.Stack;
+import java.util.Set;
public class Service {
@@ -29,14 +29,16 @@ public class Service {
private final String name;
- private final Stack stack;
+ private final StackV2 stack;
private final Configuration configuration;
+ private final ServiceGroup serviceGroup;
+
private final Set<Service> dependentServices;
- public Service(String type, Stack stack) {
- this(type, null, stack, null, null);
+ public Service(String type, StackV2 stack, ServiceGroup serviceGroup) {
+ this(type, type, stack, serviceGroup, null, null);
}
/**
@@ -46,7 +48,7 @@ public class Service {
* @param stack
* @param configuration
*/
- public Service(String type, String name, Stack stack, Configuration configuration, Set<Service> dependentServices) {
+ public Service(String type, String name, StackV2 stack, ServiceGroup serviceGroup, Configuration configuration, Set<Service> dependentServices) {
this.type = type;
if (name == null) {
this.name = type;
@@ -54,6 +56,7 @@ public class Service {
this.name = name;
}
this.stack = stack;
+ this.serviceGroup = serviceGroup;
this.configuration = configuration;
this.dependentServices = dependentServices;
}
@@ -71,11 +74,19 @@ public class Service {
return type;
}
- public Stack getStack() {
+ public StackV2 getStack() {
return stack;
}
public Configuration getConfiguration() {
return configuration;
}
+
+ public ServiceGroup getServiceGroup() {
+ return serviceGroup;
+ }
+
+ public Set<Service> getDependentServices() {
+ return dependentServices;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceGroup.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceGroup.java
index 8e66f02..4b59293 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceGroup.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceGroup.java
@@ -64,4 +64,5 @@ public class ServiceGroup {
public Set<ServiceGroup> getDependencies() {
return dependencies;
}
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index d07dec0..cfd3501 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -18,22 +18,10 @@
package org.apache.ambari.server.topology;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
+import com.google.common.eventbus.Subscribe;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.inject.persist.Transactional;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
@@ -42,21 +30,9 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.ShortTaskStatus;
-import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
-import org.apache.ambari.server.controller.internal.BaseClusterRequest;
-import org.apache.ambari.server.controller.internal.CalculatedStatus;
-import org.apache.ambari.server.controller.internal.CredentialResourceProvider;
-import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
-import org.apache.ambari.server.controller.internal.RequestImpl;
-import org.apache.ambari.server.controller.internal.ScaleClusterRequest;
-import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
-import org.apache.ambari.server.controller.spi.RequestStatus;
-import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
-import org.apache.ambari.server.controller.spi.ResourceProvider;
-import org.apache.ambari.server.controller.spi.SystemException;
-import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
+import org.apache.ambari.server.controller.internal.StackV2;
+import org.apache.ambari.server.controller.internal.*;
+import org.apache.ambari.server.controller.spi.*;
import org.apache.ambari.server.events.AmbariEvent;
import org.apache.ambari.server.events.ClusterConfigFinishedEvent;
import org.apache.ambari.server.events.HostsRemovedEvent;
@@ -79,10 +55,8 @@ import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.eventbus.Subscribe;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.google.inject.persist.Transactional;
+import java.util.*;
+import java.util.concurrent.*;
/**
* Manages all cluster provisioning actions on the cluster topology.
@@ -274,7 +248,6 @@ public class TopologyManager {
final ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request);
final String clusterName = request.getClusterName();
- final Stack stack = topology.getBlueprint().getStack();
final String repoVersion = request.getRepositoryVersion();
final Long repoVersionID = request.getRepositoryVersionId();
@@ -291,7 +264,7 @@ public class TopologyManager {
addKerberosClient(topology);
// refresh default stack config after adding KERBEROS_CLIENT component to topology
- topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getBlueprint().getServices()));
+ //topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getBlueprint().getAllServices()));
credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL);
if (credential == null) {
@@ -301,9 +274,8 @@ public class TopologyManager {
topologyValidatorService.validateTopologyConfiguration(topology);
-
// create resources
- ambariContext.createAmbariResources(topology, clusterName, securityType, repoVersion, repoVersionID);
+ ambariContext.createAmbariResources(topology, clusterName, securityType);
if (securityConfiguration != null && securityConfiguration.getDescriptor() != null) {
submitKerberosDescriptorAsArtifact(clusterName, securityConfiguration.getDescriptor());
@@ -345,8 +317,11 @@ public class TopologyManager {
//todo: this should be invoked as part of a generic lifecycle event which could possibly
//todo: be tied to cluster state
-
+ //TODO add all stack or remove concrete stack version
+ Collection<StackV2> stackList = topology.getBlueprint().getStacks();
+ StackV2 stack = stackList.iterator().next();
ambariContext.persistInstallStateForUI(clusterName, stack.getName(), stack.getVersion());
+
clusterProvisionWithBlueprintCreateRequests.put(clusterId, logicalRequest);
return getRequestStatus(logicalRequest.getRequestId());
}
@@ -1101,9 +1076,10 @@ public class TopologyManager {
* @param topology cluster topology
*/
private void addKerberosClient(ClusterTopology topology) {
- for (HostGroup group : topology.getBlueprint().getHostGroups().values()) {
- group.addComponent("KERBEROS_CLIENT");
- }
+ //TODO lookup KERBEROS_CLIENT
+// for (HostGroupV2 group : topology.getBlueprint().getHostGroups().values()) {
+// group.addComponent("KERBEROS_CLIENT");
+// }
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java
index bd5630b..632473a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
+import java.util.Collection;
import java.util.Map;
/**
@@ -52,16 +53,23 @@ public interface TopologyRequest {
*
* @return associated blueprint instance
*/
- Blueprint getBlueprint();
+ BlueprintV2 getBlueprint();
/**
* Get the cluster scoped configuration for the request.
*
* @return cluster scoped configuration
*/
+ @Deprecated
Configuration getConfiguration();
/**
+ * Returns services.
+ * @return
+ */
+ Collection<Service> getServiceConfigs();
+
+ /**
* Get host group info.
*
* @return map of host group name to group info
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
index 990aee7..f8ad78e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java
@@ -17,20 +17,17 @@
*/
package org.apache.ambari.server.topology.tasks;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+import org.apache.ambari.server.topology.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.HostRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.assistedinject.Assisted;
-import com.google.inject.assistedinject.AssistedInject;
-
public class PersistHostResourcesTask extends TopologyHostTask {
private final static Logger LOG = LoggerFactory.getLogger(PersistHostResourcesTask.class);
@@ -49,10 +46,10 @@ public class PersistHostResourcesTask extends TopologyHostTask {
public void runTask() {
LOG.info("HostRequest: Executing RESOURCE_CREATION task for host: {}", hostRequest.getHostName());
- HostGroup group = hostRequest.getHostGroup();
- Map<String, Collection<String>> serviceComponents = new HashMap<>();
- for (String service : group.getServices()) {
- serviceComponents.put(service, new HashSet<>(group.getComponents(service)));
+ HostGroupV2 group = hostRequest.getHostGroup();
+ Map<Service, Collection<ComponentV2>> serviceComponents = new HashMap<>();
+ for (Service service : group.getServices()) {
+ serviceComponents.put(service, new HashSet(group.getComponents(service)));
}
clusterTopology.getAmbariContext().createAmbariHostResources(hostRequest.getClusterId(),
hostRequest.getHostName(), serviceComponents);
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java
index 0170186..4895c40 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java
@@ -13,15 +13,16 @@
*/
package org.apache.ambari.server.topology.validators;
-import java.util.HashSet;
-import java.util.Set;
-
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.Service;
import org.apache.ambari.server.topology.TopologyValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* Validates configuration types related to services specified in the blueprint.
* If the cluster creation template contains configuration types that are not related to services in the blueprint the
@@ -44,8 +45,8 @@ public class ClusterConfigTypeValidator implements TopologyValidator {
// collecting all config types for services in the blueprint (from the related stack)
Set<String> stackServiceConfigTypes = new HashSet<>();
- for (String serviceName : topology.getBlueprint().getServices()) {
- stackServiceConfigTypes.addAll(topology.getBlueprint().getStack().getConfigurationTypes(serviceName));
+ for (Service service : topology.getBlueprint().getAllServices()) {
+ stackServiceConfigTypes.addAll(service.getStack().getConfigurationTypes(service.getType()));
}
// identifying invalid config types
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
index 80b2593..3f8b517 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
@@ -14,13 +14,12 @@
package org.apache.ambari.server.topology.validators;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.InvalidTopologyException;
-import org.apache.ambari.server.topology.TopologyValidator;
+import org.apache.ambari.server.topology.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+
/**
* Validates hive service related setup before provisioning the cluster.
*/
@@ -37,34 +36,41 @@ public class HiveServiceValidator implements TopologyValidator {
@Override
public void validate(ClusterTopology topology) throws InvalidTopologyException {
+ Collection<Service> services = topology.getBlueprint().getServicesByType(HIVE_SERVICE);
// there is no hive configured in the blueprint, nothing to do (does the validator apply?)
- if (!topology.getBlueprint().getServices().contains(HIVE_SERVICE)) {
+ if (services.isEmpty()) {
LOGGER.info(" [{}] service is not listed in the blueprint, skipping hive service validation.", HIVE_SERVICE);
return;
}
- Configuration clusterConfiguration = topology.getConfiguration();
+ for (Service service : services) {
- // hive database settings are missing (this should never be the case, defaults come from the stack def.)
- if (!clusterConfiguration.getAllConfigTypes().contains(HIVE_ENV)) {
- String errorMessage = String.format(" [ %s ] config type is missing from the service [ %s ]. HIVE service validation failed.", HIVE_ENV, HIVE_SERVICE);
- LOGGER.error(errorMessage);
- throw new InvalidTopologyException(errorMessage);
- }
+ Configuration serviceConfiguration = service.getConfiguration();
- // hive database has custom configuration, skipping validation
- if (!HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY))) {
- LOGGER.info("Custom hive database settings detected. HIVE service validation succeeded.");
- return;
- }
+ // hive database settings are missing (this should never be the case, defaults come from the stack def.)
+ if (!serviceConfiguration.getAllConfigTypes().contains(HIVE_ENV)) {
+ String errorMessage = String.format(" [ %s ] config type is missing from the service [ %s ]. HIVE service validation failed.", HIVE_ENV, HIVE_SERVICE);
+ LOGGER.error(errorMessage);
+ throw new InvalidTopologyException(errorMessage);
+ }
- // hive database settings need the mysql-server component in the blueprint
- if (!topology.getBlueprint().getComponents(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) {
- String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " +
+ // hive database has custom configuration, skipping validation
+ if (!HIVE_DB_DEFAULT.equals(serviceConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY))) {
+ LOGGER.info("Custom hive database settings detected. HIVE service validation succeeded.");
+ return;
+ }
+
+ Collection<ComponentV2> mySqlComponents = topology.getBlueprint().getComponentsByType(service, MYSQL_SERVER_COMPONENT);
+
+ // hive database settings need the mysql-server component in the blueprint
+ if (mySqlComponents.isEmpty()) {
+ String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " +
"is configured with the current settings. HIVE service validation failed.", MYSQL_SERVER_COMPONENT);
- LOGGER.error(errorMessage);
- throw new InvalidTopologyException(errorMessage);
+ LOGGER.error(errorMessage);
+ throw new InvalidTopologyException(errorMessage);
+ }
}
+
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
index 4022fcb..4d37a02 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java
@@ -14,23 +14,14 @@
package org.apache.ambari.server.topology.validators;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.internal.StackV2;
import org.apache.ambari.server.state.PropertyInfo;
-import org.apache.ambari.server.topology.Blueprint;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.InvalidTopologyException;
-import org.apache.ambari.server.topology.TopologyValidator;
+import org.apache.ambari.server.topology.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+
/**
* Validates the configuration by checking the existence of required properties for the services listed in the blueprint.
@@ -52,14 +43,11 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator {
@Override
public void validate(ClusterTopology topology) throws InvalidTopologyException {
- // collect required properties
- Map<String, Map<String, Collection<String>>> requiredPropertiesByService = getRequiredPropertiesByService(topology.getBlueprint());
-
// find missing properties in the cluster configuration
Map<String, Collection<String>> missingProperties = new TreeMap<>();
Map<String, Map<String, String>> topologyConfiguration = new HashMap<>(topology.getConfiguration().getFullProperties(1));
- for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) {
+ for (HostGroupV2 hostGroup : topology.getBlueprint().getHostGroups().values()) {
LOGGER.debug("Processing hostgroup configurations for hostgroup: {}", hostGroup.getName());
// copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig
@@ -73,21 +61,21 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator {
}
}
- for (String hostGroupService : hostGroup.getServices()) {
+ for (Service hostGroupService : hostGroup.getServices()) {
- if (!requiredPropertiesByService.containsKey(hostGroupService)) {
+ // collect required properties
+ Map<String, Collection<String>> requiredPropertiesForService = getRequiredPropertiesForService(hostGroupService);
+ if (requiredPropertiesForService.isEmpty()) {
// there are no required properties for the service
LOGGER.debug("There are no required properties found for hostgroup/service: [{}/{}]", hostGroup.getName(), hostGroupService);
continue;
}
- Map<String, Collection<String>> requiredPropertiesByType = requiredPropertiesByService.get(hostGroupService);
-
- for (String configType : requiredPropertiesByType.keySet()) {
+ for (String configType : requiredPropertiesForService.keySet()) {
// We need a copy not to modify the original
Collection<String> requiredPropertiesForType = new HashSet(
- requiredPropertiesByType.get(configType));
+ requiredPropertiesForService.get(configType));
if (!operationalConfigurations.containsKey(configType)) {
// all required configuration is missing for the config type
@@ -116,60 +104,48 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator {
/**
- * Collects required properties for services in the blueprint. Configuration properties are returned by configuration type.
- * service -> configType -> properties
- *
- * @param blueprint the blueprint from the cluster topology
+ * Collects required properties for a specified services in the blueprint. Configuration properties are returned
+ * by configuration type. configType -> properties
+ * @param service the blueprint from the cluster topology
* @return a map with configuration types mapped to collections of required property names
*/
- private Map<String, Map<String, Collection<String>>> getRequiredPropertiesByService(Blueprint blueprint) {
+ private Map<String, Collection<String>> getRequiredPropertiesForService(Service service) {
- Map<String, Map<String, Collection<String>>> requiredPropertiesForServiceByType = new HashMap<>();
+ LOGGER.debug("Collecting required properties for the service: {}", service.getName());
- for (String bpService : blueprint.getServices()) {
- LOGGER.debug("Collecting required properties for the service: {}", bpService);
+ Collection<StackV2.ConfigProperty> requiredConfigsForService = service.getStack().
+ getRequiredConfigurationProperties(service.getType());
+ Map<String, Collection<String>> requiredPropertiesByConfigType = new HashMap<>();
- Collection<Stack.ConfigProperty> requiredConfigsForService = blueprint.getStack().getRequiredConfigurationProperties(bpService);
- Map<String, Collection<String>> requiredPropertiesByConfigType = new HashMap<>();
-
- for (Stack.ConfigProperty configProperty : requiredConfigsForService) {
-
- if (configProperty.getPropertyTypes() != null && configProperty.getPropertyTypes().contains(PropertyInfo.PropertyType.PASSWORD)) {
- LOGGER.debug("Skipping required property validation for password type: {}", configProperty.getName());
- // skip password types
- continue;
- }
+ for (StackV2.ConfigProperty configProperty : requiredConfigsForService) {
- // add / get service related required propeByType map
- if (requiredPropertiesForServiceByType.containsKey(bpService)) {
- requiredPropertiesByConfigType = requiredPropertiesForServiceByType.get(bpService);
- } else {
- LOGGER.debug("Adding required properties entry for service: {}", bpService);
- requiredPropertiesForServiceByType.put(bpService, requiredPropertiesByConfigType);
- }
-
- // add collection of required properties
- Collection<String> requiredPropsForType = new HashSet<>();
- if (requiredPropertiesByConfigType.containsKey(configProperty.getType())) {
- requiredPropsForType = requiredPropertiesByConfigType.get(configProperty.getType());
- } else {
- LOGGER.debug("Adding required properties entry for configuration type: {}", configProperty.getType());
- requiredPropertiesByConfigType.put(configProperty.getType(), requiredPropsForType);
- }
+ if (configProperty.getPropertyTypes() != null && configProperty.getPropertyTypes().contains(PropertyInfo.PropertyType.PASSWORD)) {
+ LOGGER.debug("Skipping required property validation for password type: {}", configProperty.getName());
+ // skip password types
+ continue;
+ }
- requiredPropsForType.add(configProperty.getName());
- LOGGER.debug("Added required property for service; {}, configuration type: {}, property: {}", bpService,
- configProperty.getType(), configProperty.getName());
+ // add collection of required properties
+ Collection<String> requiredPropsForType = new HashSet<>();
+ if (requiredPropertiesByConfigType.containsKey(configProperty.getType())) {
+ requiredPropsForType = requiredPropertiesByConfigType.get(configProperty.getType());
+ } else {
+ LOGGER.debug("Adding required properties entry for configuration type: {}", configProperty.getType());
+ requiredPropertiesByConfigType.put(configProperty.getType(), requiredPropsForType);
}
+
+ requiredPropsForType.add(configProperty.getName());
+ LOGGER.debug("Added required property for service; {}, configuration type: {}, property: {}", service.getName(),
+ configProperty.getType(), configProperty.getName());
}
- LOGGER.info("Identified required properties for blueprint services: {}", requiredPropertiesForServiceByType);
- return requiredPropertiesForServiceByType;
+ return requiredPropertiesByConfigType;
}
- private Map<String, Collection<String>> addTomissingProperties(Map<String, Collection<String>> missingProperties, String hostGroup, Collection<String> values) {
+ private Map<String, Collection<String>> addTomissingProperties(Map<String, Collection<String>> missingProperties,
+ String hostGroup, Collection<String> values) {
Map<String, Collection<String>> missing;
if (missingProperties == null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java
index 5b4ecc1..9a2b846 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java
@@ -14,20 +14,15 @@ package org.apache.ambari.server.topology.validators;
* limitations under the License.
*/
+import org.apache.ambari.server.controller.internal.StackV2;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.topology.*;
+
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.state.PropertyInfo;
-import org.apache.ambari.server.topology.Blueprint;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.HostGroupInfo;
-import org.apache.ambari.server.topology.InvalidTopologyException;
-import org.apache.ambari.server.topology.TopologyValidator;
-
/**
* Validates that all required passwords are provided.
*/
@@ -80,23 +75,22 @@ public class RequiredPasswordValidator implements TopologyValidator {
groupEntry.getValue().getConfiguration().getFullProperties(3);
Collection<String> processedServices = new HashSet<>();
- Blueprint blueprint = topology.getBlueprint();
- Stack stack = blueprint.getStack();
+ BlueprintV2 blueprint = topology.getBlueprint();
- HostGroup hostGroup = blueprint.getHostGroup(hostGroupName);
- for (String component : hostGroup.getComponentNames()) {
+ HostGroupV2 hostGroup = blueprint.getHostGroup(hostGroupName);
+ for (ComponentV2 component : hostGroup.getComponents()) {
//for now, AMBARI is not recognized as a service in Stacks
- if (component.equals("AMBARI_SERVER")) {
+ if (component.getType().equals("AMBARI_SERVER")) {
continue;
}
- String serviceName = stack.getServiceForComponent(component);
- if (processedServices.add(serviceName)) {
+ Service service = component.getService();
+ if (processedServices.add(service.getName())) {
//todo: do I need to subtract excluded configs?
- Collection<Stack.ConfigProperty> requiredProperties =
- stack.getRequiredConfigurationProperties(serviceName, PropertyInfo.PropertyType.PASSWORD);
+ Collection<StackV2.ConfigProperty> requiredProperties =
+ service.getStack().getRequiredConfigurationProperties(service.getType(), PropertyInfo.PropertyType.PASSWORD);
- for (Stack.ConfigProperty property : requiredProperties) {
+ for (StackV2.ConfigProperty property : requiredProperties) {
String category = property.getType();
String name = property.getName();
if (! propertyExists(topology, groupProperties, category, name)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
index f028a31..55660e3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
@@ -14,15 +14,16 @@
package org.apache.ambari.server.topology.validators;
-import java.util.HashSet;
-import java.util.Set;
-
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.Service;
import org.apache.ambari.server.topology.TopologyValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* Validates whether incoming config types (form the blueprint or the cluster creation template) are valid.
* A configuration type is considered valid if the stack based on which the cluster is to be created contains such a
@@ -36,25 +37,27 @@ public class StackConfigTypeValidator implements TopologyValidator {
@Override
public void validate(ClusterTopology topology) throws InvalidTopologyException {
+ for (Service service : topology.getServiceConfigs()) {
+ // get the config types form the request
+ Set<String> incomingConfigTypes = new HashSet<>(service.getConfiguration().getAllConfigTypes());
- // get the config types form the request
- Set<String> incomingConfigTypes = new HashSet<>(topology.getConfiguration().getAllConfigTypes());
+ if (incomingConfigTypes.isEmpty()) {
+ LOGGER.debug("No config types to be checked.");
+ return;
+ }
- if (incomingConfigTypes.isEmpty()) {
- LOGGER.debug("No config types to be checked.");
- return;
- }
+ Set<String> stackConfigTypes = new HashSet<>(service.getStack().getConfiguration().getAllConfigTypes());
- Set<String> stackConfigTypes = new HashSet<>(topology.getBlueprint().getStack().getConfiguration().getAllConfigTypes());
+ // remove all "valid" config types from the incoming set
+ incomingConfigTypes.removeAll(stackConfigTypes);
- // remove all "valid" config types from the incoming set
- incomingConfigTypes.removeAll(stackConfigTypes);
+ if (!incomingConfigTypes.isEmpty()) {
+ // there are config types in the request that are not in the stack
+ String message = String.format("The following config types are not defined in the stack: %s ", incomingConfigTypes);
+ LOGGER.error(message);
+ throw new InvalidTopologyException(message);
+ }
- if (!incomingConfigTypes.isEmpty()) {
- // there are config types in the request that are not in the stack
- String message = String.format("The following config types are not defined in the stack: %s ", incomingConfigTypes);
- LOGGER.error(message);
- throw new InvalidTopologyException(message);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java
index e75ffa4..50bdec6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java
@@ -17,18 +17,17 @@
*/
package org.apache.ambari.server.topology.validators;
-import static org.apache.ambari.server.controller.internal.UnitUpdater.PropertyValue;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.internal.StackV2;
import org.apache.ambari.server.controller.internal.UnitUpdater.PropertyUnit;
import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.HostGroupInfo;
import org.apache.ambari.server.topology.InvalidTopologyException;
import org.apache.ambari.server.topology.TopologyValidator;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.ambari.server.controller.internal.UnitUpdater.PropertyValue;
+
/**
* I validate the unit of properties by checking if it matches to the stack defined unit.
* Properties with different unit than the stack defined unit are rejected.
@@ -42,33 +41,32 @@ public class UnitValidator implements TopologyValidator {
@Override
public void validate(ClusterTopology topology) throws InvalidTopologyException {
- Stack stack = topology.getBlueprint().getStack();
- validateConfig(topology.getConfiguration().getFullProperties(), stack);
- for (HostGroupInfo hostGroup : topology.getHostGroupInfo().values()) {
- validateConfig(hostGroup.getConfiguration().getFullProperties(), stack);
- }
+ topology.getServiceConfigs().forEach(service -> {
+ validateConfig(service.getConfiguration().getFullProperties(), service.getStack());
+ topology.getHostGroupInfo().values().forEach(hostGroup ->
+ validateConfig(hostGroup.getConfiguration().getFullProperties(), service.getStack()));
+ });
+
}
- private void validateConfig(Map<String, Map<String, String>> configuration, Stack stack) {
- for (Map.Entry<String, Map<String, String>> each : configuration.entrySet()) {
- validateConfigType(each.getKey(), each.getValue(), stack);
- }
+ private void validateConfig(Map<String, Map<String, String>> configuration, StackV2 stack) {
+ configuration.entrySet().forEach(each ->
+ validateConfigType(each.getKey(), each.getValue(), stack)
+ );
}
- private void validateConfigType(String configType, Map<String, String> config, Stack stack) {
- for (String propertyName : config.keySet()) {
- validateProperty(configType, config, propertyName, stack);
- }
+ private void validateConfigType(String configType, Map<String, String> config, StackV2 stack) {
+ config.keySet().forEach(propertyName -> validateProperty(configType, config, propertyName, stack));
}
- private void validateProperty(String configType, Map<String, String> config, String propertyName, Stack stack) {
+ private void validateProperty(String configType, Map<String, String> config, String propertyName, StackV2 stack) {
relevantProps.stream()
.filter(each -> each.hasTypeAndName(configType, propertyName))
.findFirst()
.ifPresent(relevantProperty -> checkUnit(config, stack, relevantProperty));
}
- private void checkUnit(Map<String, String> configToBeValidated, Stack stack, UnitValidatedProperty prop) {
+ private void checkUnit(Map<String, String> configToBeValidated, StackV2 stack, UnitValidatedProperty prop) {
PropertyUnit stackUnit = PropertyUnit.of(stack, prop);
PropertyValue value = PropertyValue.of(prop.getPropertyName(), configToBeValidated.get(prop.getPropertyName()));
if (value.hasAnyUnit() && !value.hasUnit(stackUnit)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
index 8c44632..bbd0fea 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
@@ -322,7 +322,7 @@ public class AmbariMetaInfoTest {
}
/**
- * Method: Map<String, ServiceInfo> getServices(String stackName, String
+ * Method: Map<String, ServiceInfo> getServiceConfigs(String stackName, String
* version, String serviceName)
* @throws AmbariException
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/api/services/RootServiceServiceTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/RootServiceServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/RootServiceServiceTest.java
index cbf2036..490d2da 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/RootServiceServiceTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/RootServiceServiceTest.java
@@ -40,7 +40,7 @@ public class RootServiceServiceTest extends BaseServiceTest {
public List<ServiceTestInvocation> getTestInvocations() throws Exception {
List<ServiceTestInvocation> listInvocations = new ArrayList<>();
- //getServices
+ //getServiceConfigs
RootServiceService service = new TestRootServiceService(null, null, null);
Method m = service.getClass().getMethod("getRootServices", String.class, HttpHeaders.class, UriInfo.class);
Object[] args = new Object[] {null, getHttpHeaders(), getUriInfo()};
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/api/services/ServiceServiceTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/ServiceServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/ServiceServiceTest.java
index a51d710..a5bb5e7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/ServiceServiceTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/ServiceServiceTest.java
@@ -45,7 +45,7 @@ public class ServiceServiceTest extends BaseServiceTest {
Object[] args = new Object[] {null, getHttpHeaders(), getUriInfo(), "serviceName"};
listInvocations.add(new ServiceTestInvocation(Request.Type.GET, service, m, args, null));
- //getServices
+ //getServiceConfigs
service = new TestServiceService("clusterName", null);
m = service.getClass().getMethod("getServices", String.class, HttpHeaders.class, UriInfo.class);
args = new Object[] {null, getHttpHeaders(), getUriInfo()};
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index bf2b84d..44b46f4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -8961,7 +8961,7 @@ public class AmbariManagementControllerTest {
expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(maintHelper);
expect(injector.getInstance(KerberosHelper.class)).andReturn(createStrictMock(KerberosHelper.class));
- // getServices
+ // getServiceConfigs
expect(clusters.getCluster("cluster1")).andReturn(cluster);
expect(cluster.getService("service1")).andReturn(service);
@@ -9005,7 +9005,7 @@ public class AmbariManagementControllerTest {
expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(maintHelper);
expect(injector.getInstance(KerberosHelper.class)).andReturn(createStrictMock(KerberosHelper.class));
- // getServices
+ // getServiceConfigs
expect(clusters.getCluster("cluster1")).andReturn(cluster);
expect(cluster.getService("service1")).andThrow(new ServiceNotFoundException("custer1", "service1"));
@@ -9064,7 +9064,7 @@ public class AmbariManagementControllerTest {
expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(maintHelper);
expect(injector.getInstance(KerberosHelper.class)).andReturn(createStrictMock(KerberosHelper.class));
- // getServices
+ // getServiceConfigs
expect(clusters.getCluster("cluster1")).andReturn(cluster).times(4);
expect(cluster.getService("service1")).andReturn(service1);
expect(cluster.getService("service2")).andThrow(new ServiceNotFoundException("cluster1", "service2"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
index 4b09c6d..c9e3ce9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
@@ -599,7 +599,7 @@ public class ClusterTest {
// TODO write unit tests for
// public void addService(Service service) throws AmbariException;
// public Service getService(String serviceName) throws AmbariException;
- // public Map<String, Service> getServices();
+ // public Map<String, Service> getServiceConfigs();
RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(c1);
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index 6ac74a3..bb33edd 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -459,7 +459,7 @@ public class BlueprintImplTest {
// expect(request.getProperties()).andReturn(setProperties);
// expect(request.getRequestInfoProperties()).andReturn(Collections.<String, String>emptyMap());
// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null);
-// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
+// expect(metaInfo.getServiceConfigs("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")).
// andReturn(serviceComponents).anyTimes();
// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")).
@@ -570,7 +570,7 @@ public class BlueprintImplTest {
// expect(request.getProperties()).andReturn(setProperties);
// expect(request.getRequestInfoProperties()).andReturn(Collections.<String, String>emptyMap());
// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null);
-// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
+// expect(metaInfo.getServiceConfigs("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")).
// andReturn(serviceComponents).anyTimes();
// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")).
@@ -689,7 +689,7 @@ public class BlueprintImplTest {
// expect(request.getProperties()).andReturn(setProperties);
// expect(request.getRequestInfoProperties()).andReturn(Collections.<String, String>emptyMap());
// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null);
-// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
+// expect(metaInfo.getServiceConfigs("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")).
// andReturn(serviceComponents).anyTimes();
// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")).
@@ -797,7 +797,7 @@ public class BlueprintImplTest {
// expect(request.getProperties()).andReturn(setProperties);
// expect(request.getRequestInfoProperties()).andReturn(Collections.<String, String>emptyMap());
// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null);
-// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
+// expect(metaInfo.getServiceConfigs("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")).
// andReturn(serviceComponents).anyTimes();
// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")).
@@ -865,7 +865,7 @@ public class BlueprintImplTest {
// expect(request.getProperties()).andReturn(setProperties);
// expect(request.getRequestInfoProperties()).andReturn(Collections.<String, String>emptyMap());
// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null);
-// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
+// expect(metaInfo.getServiceConfigs("test-stack-name", "test-stack-version")).andReturn(services).anyTimes();
// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")).
// andReturn(serviceComponents).anyTimes();
// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")).
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
index e51ce5f..87d6363 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
@@ -18,23 +18,16 @@
package org.apache.ambari.server.topology;
-import static org.easymock.EasyMock.expect;
-import static org.powermock.api.easymock.PowerMock.createNiceMock;
-import static org.powermock.api.easymock.PowerMock.replay;
-import static org.powermock.api.easymock.PowerMock.reset;
-import static org.powermock.api.easymock.PowerMock.verify;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
+import org.apache.ambari.server.controller.internal.Stack;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.util.*;
+
+import static org.easymock.EasyMock.expect;
+import static org.powermock.api.easymock.PowerMock.*;
+
/**
* Unit tests for ClusterTopologyImpl.
*/
@@ -44,13 +37,14 @@ public class ClusterTopologyImplTest {
private static final String CLUSTER_NAME = "cluster_name";
private static final long CLUSTER_ID = 1L;
private static final String predicate = "Hosts/host_name=foo";
- private static final Blueprint blueprint = createNiceMock(Blueprint.class);
- private static final HostGroup group1 = createNiceMock(HostGroup.class);
- private static final HostGroup group2 = createNiceMock(HostGroup.class);
- private static final HostGroup group3 = createNiceMock(HostGroup.class);
- private static final HostGroup group4 = createNiceMock(HostGroup.class);
+ private static final BlueprintV2 blueprint = createNiceMock(BlueprintV2.class);
+ private static final HostGroupV2 group1 = createNiceMock(HostGroupV2.class);
+ private static final HostGroupV2 group2 = createNiceMock(HostGroupV2.class);
+ private static final HostGroupV2 group3 = createNiceMock(HostGroupV2.class);
+ private static final HostGroupV2 group4 = createNiceMock(HostGroupV2.class);
+ private static final Stack stack = createNiceMock(Stack.class);
private final Map<String, HostGroupInfo> hostGroupInfoMap = new HashMap<>();
- private final Map<String, HostGroup> hostGroupMap = new HashMap<>();
+ private final Map<String, HostGroupV2> hostGroupMap = new HashMap<>();
private Configuration configuration;
private Configuration bpconfiguration;
@@ -101,20 +95,20 @@ public class ClusterTopologyImplTest {
hostGroupMap.put("group3", group3);
hostGroupMap.put("group4", group4);
- Set<Component> group1Components = new HashSet<>();
- group1Components.add(new Component("component1"));
- group1Components.add(new Component("component2"));
+ Set<ComponentV2> group1Components = new HashSet<>();
+ group1Components.add(new ComponentV2("component1", new Service("service1", stack)));
+ group1Components.add(new ComponentV2("component2", new Service("service1", stack)));
Set<String> group1ComponentNames = new HashSet<>();
group1ComponentNames.add("component1");
group1ComponentNames.add("component2");
- Set<Component> group2Components = new HashSet<>();
- group2Components.add(new Component("component3"));
- Set<Component> group3Components = new HashSet<>();
- group3Components.add(new Component("component4"));
- Set<Component> group4Components = new HashSet<>();
- group4Components.add(new Component("component5"));
+ Set<ComponentV2> group2Components = new HashSet<>();
+ group2Components.add(new ComponentV2("component3", new Service("service1", stack)));
+ Set<ComponentV2> group3Components = new HashSet<>();
+ group3Components.add(new ComponentV2("component4", new Service("service2", stack)));
+ Set<ComponentV2> group4Components = new HashSet<>();
+ group4Components.add(new ComponentV2("component5", new Service("service2", stack)));
expect(blueprint.getHostGroups()).andReturn(hostGroupMap).anyTimes();
expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes();
@@ -143,7 +137,6 @@ public class ClusterTopologyImplTest {
verify(blueprint, group1, group2, group3, group4);
reset(blueprint, group1, group2, group3, group4);
-
hostGroupInfoMap.clear();
hostGroupMap.clear();
}
@@ -241,7 +234,7 @@ public class ClusterTopologyImplTest {
}
@Override
- public Blueprint getBlueprint() {
+ public BlueprintV2 getBlueprint() {
return blueprint;
}
@@ -251,6 +244,11 @@ public class ClusterTopologyImplTest {
}
@Override
+ public Collection<Service> getServiceConfigs() {
+ return null;
+ }
+
+ @Override
public Map<String, HostGroupInfo> getHostGroupInfo() {
return hostGroupInfoMap;
}
[3/4] ambari git commit: use V2 objects in TopologyManager
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 03f84a5..4c51762 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -37,30 +37,19 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.state.PropertyDependencyInfo;
-import org.apache.ambari.server.state.StackId;
-import org.apache.ambari.server.state.ValueAttributesInfo;
-import org.apache.ambari.server.topology.AdvisedConfiguration;
-import org.apache.ambari.server.topology.Blueprint;
-import org.apache.ambari.server.topology.Cardinality;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
-import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.HostGroupInfo;
+import org.apache.ambari.server.state.*;
+import org.apache.ambari.server.topology.*;
+import org.apache.ambari.server.topology.Service;
import org.apache.ambari.server.topology.validators.UnitValidatedProperty;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
/**
* Updates configuration properties based on cluster topology. This is done when exporting
* a blueprint and when a cluster is provisioned via a blueprint.
@@ -213,10 +202,11 @@ public class BlueprintConfigurationProcessor {
new HawqHAFilter() };
private ClusterTopology clusterTopology;
+ public final ConfigurationContext configurationContext;
-
- public BlueprintConfigurationProcessor(ClusterTopology clusterTopology) {
+ public BlueprintConfigurationProcessor(ClusterTopology clusterTopology, ConfigurationContext configurationContext) {
this.clusterTopology = clusterTopology;
+ this.configurationContext = configurationContext;
initRemovePropertyUpdaters();
}
@@ -309,7 +299,7 @@ public class BlueprintConfigurationProcessor {
Map<String, String> typeMap = clusterProps.get(type);
if (typeMap != null && typeMap.containsKey(propertyName) && typeMap.get(propertyName) != null) {
requiredHostGroups.addAll(updater.getRequiredHostGroups(
- propertyName, typeMap.get(propertyName), clusterProps, clusterTopology));
+ propertyName, typeMap.get(propertyName), clusterProps, clusterTopology, configurationContext));
}
// host group configs
@@ -318,7 +308,7 @@ public class BlueprintConfigurationProcessor {
Map<String, String> hgTypeMap = hgConfigProps.get(type);
if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) {
requiredHostGroups.addAll(updater.getRequiredHostGroups(
- propertyName, hgTypeMap.get(propertyName), hgConfigProps, clusterTopology));
+ propertyName, hgTypeMap.get(propertyName), hgConfigProps, clusterTopology, configurationContext));
}
}
}
@@ -361,7 +351,7 @@ public class BlueprintConfigurationProcessor {
if (typeMap != null && typeMap.containsKey(propertyName) && typeMap.get(propertyName) != null) {
final String originalValue = typeMap.get(propertyName);
final String updatedValue =
- updater.updateForClusterCreate(propertyName, originalValue, clusterProps, clusterTopology);
+ updater.updateForClusterCreate(propertyName, originalValue, clusterProps, clusterTopology, configurationContext);
if(updatedValue == null ) {
continue;
@@ -382,7 +372,7 @@ public class BlueprintConfigurationProcessor {
if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) {
final String originalValue = hgTypeMap.get(propertyName);
final String updatedValue =
- updater.updateForClusterCreate(propertyName, originalValue, hgConfigProps, clusterTopology);
+ updater.updateForClusterCreate(propertyName, originalValue, hgConfigProps, clusterTopology, configurationContext);
if (!updatedValue.equals(originalValue)) {
configTypesUpdated.add(type);
@@ -396,7 +386,7 @@ public class BlueprintConfigurationProcessor {
}
//todo: lots of hard coded HA rules included here
- if (clusterTopology.isNameNodeHAEnabled()) {
+ if (clusterTopology.isNameNodeHAEnabled(configurationContext)) {
// add "dfs.internal.nameservices" if it's not specified
Map<String, String> hdfsSiteConfig = clusterConfig.getFullProperties().get("hdfs-site");
@@ -428,7 +418,7 @@ public class BlueprintConfigurationProcessor {
setStackToolsAndFeatures(clusterConfig, configTypesUpdated);
setRetryConfiguration(clusterConfig, configTypesUpdated);
setupHDFSProxyUsers(clusterConfig, configTypesUpdated);
- addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getBlueprint().getStack());
+ addExcludedConfigProperties(clusterConfig, configTypesUpdated);
trimProperties(clusterConfig, clusterTopology);
@@ -436,15 +426,15 @@ public class BlueprintConfigurationProcessor {
}
private void trimProperties(Configuration clusterConfig, ClusterTopology clusterTopology) {
- Blueprint blueprint = clusterTopology.getBlueprint();
- Stack stack = blueprint.getStack();
+ BlueprintV2 blueprint = clusterTopology.getBlueprint();
+ //Stack stack = blueprint.getStack();
Map<String, Map<String, String>> configTypes = clusterConfig.getFullProperties();
for (String configType : configTypes.keySet()) {
Map<String,String> properties = configTypes.get(configType);
- for (String propertyName : properties.keySet()) {
- trimPropertyValue(clusterConfig, stack, configType, properties, propertyName);
- }
+// for (String propertyName : properties.keySet()) {
+// trimPropertyValue(clusterConfig, stack, configType, properties, propertyName);
+// }
}
}
@@ -485,11 +475,11 @@ public class BlueprintConfigurationProcessor {
*/
public void doUpdateForBlueprintExport() {
// HA configs are only processed in cluster configuration, not HG configurations
- if (clusterTopology.isNameNodeHAEnabled()) {
+ if (clusterTopology.isNameNodeHAEnabled(configurationContext)) {
doNameNodeHAUpdate();
}
- if (clusterTopology.isYarnResourceManagerHAEnabled()) {
+ if (clusterTopology.isYarnResourceManagerHAEnabled(configurationContext)) {
doYarnResourceManagerHAUpdate();
}
@@ -547,7 +537,7 @@ public class BlueprintConfigurationProcessor {
for (Map.Entry<String, Map<String, String>> configEntry : properties.entrySet()) {
String type = configEntry.getKey();
try {
- clusterTopology.getBlueprint().getStack().getServiceForConfigType(type);
+ // clusterTopology.getBlueprint().getStack().getServiceForConfigType(type);
} catch (IllegalArgumentException illegalArgumentException) {
LOG.error(new StringBuilder(String.format("Error encountered while trying to obtain the service name for config type [%s]. ", type))
.append("Further processing on this config type will be skipped. ")
@@ -576,7 +566,7 @@ public class BlueprintConfigurationProcessor {
for (Map.Entry<String, String> propertyEntry : configPropertiesPerType.entrySet()) {
String propName = propertyEntry.getKey();
- if (shouldPropertyBeExcludedForClusterUpdate(propName, propertyEntry.getValue(), configType, clusterTopology)) {
+ if (shouldPropertyBeExcludedForClusterUpdate(propName, propertyEntry.getValue(), configType, clusterTopology, configurationContext)) {
configuration.removeProperty(configType, propName);
configTypesUpdated.add(configType);
}
@@ -621,29 +611,29 @@ public class BlueprintConfigurationProcessor {
* @param advisedConfigurations advised configuration instance
*/
private void doFilterStackDefaults(Map<String, AdvisedConfiguration> advisedConfigurations) {
- Blueprint blueprint = clusterTopology.getBlueprint();
- Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
- Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
- for (Map.Entry<String, AdvisedConfiguration> adConfEntry : advisedConfigurations.entrySet()) {
- AdvisedConfiguration advisedConfiguration = adConfEntry.getValue();
- if (stackDefaultProps.containsKey(adConfEntry.getKey())) {
- Map<String, String> defaultProps = stackDefaultProps.get(adConfEntry.getKey());
- if (advisedConfiguration.getProperties() != null) {
- Map<String, String> outFilteredProps = Maps.filterKeys(advisedConfiguration.getProperties(),
- Predicates.not(Predicates.in(defaultProps.keySet())));
- advisedConfiguration.getProperties().keySet().removeAll(Sets.newCopyOnWriteArraySet(outFilteredProps.keySet()));
- }
-
- if (advisedConfiguration.getPropertyValueAttributes() != null) {
- Map<String, ValueAttributesInfo> outFilteredValueAttrs = Maps.filterKeys(advisedConfiguration.getPropertyValueAttributes(),
- Predicates.not(Predicates.in(defaultProps.keySet())));
- advisedConfiguration.getPropertyValueAttributes().keySet().removeAll(
- Sets.newCopyOnWriteArraySet(outFilteredValueAttrs.keySet()));
- }
- } else {
- advisedConfiguration.getProperties().clear();
- }
- }
+// BlueprintV2 blueprint = clusterTopology.getBlueprint();
+// Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
+// Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
+// for (Map.Entry<String, AdvisedConfiguration> adConfEntry : advisedConfigurations.entrySet()) {
+// AdvisedConfiguration advisedConfiguration = adConfEntry.getValue();
+// if (stackDefaultProps.containsKey(adConfEntry.getKey())) {
+// Map<String, String> defaultProps = stackDefaultProps.get(adConfEntry.getKey());
+// if (advisedConfiguration.getProperties() != null) {
+// Map<String, String> outFilteredProps = Maps.filterKeys(advisedConfiguration.getProperties(),
+// Predicates.not(Predicates.in(defaultProps.keySet())));
+// advisedConfiguration.getProperties().keySet().removeAll(Sets.newCopyOnWriteArraySet(outFilteredProps.keySet()));
+// }
+//
+// if (advisedConfiguration.getPropertyValueAttributes() != null) {
+// Map<String, ValueAttributesInfo> outFilteredValueAttrs = Maps.filterKeys(advisedConfiguration.getPropertyValueAttributes(),
+// Predicates.not(Predicates.in(defaultProps.keySet())));
+// advisedConfiguration.getPropertyValueAttributes().keySet().removeAll(
+// Sets.newCopyOnWriteArraySet(outFilteredValueAttrs.keySet()));
+// }
+// } else {
+// advisedConfiguration.getProperties().clear();
+// }
+// }
}
/**
@@ -700,11 +690,11 @@ public class BlueprintConfigurationProcessor {
private Collection<Map<String, Map<String, PropertyUpdater>>> createCollectionOfUpdaters() {
Collection<Map<String, Map<String, PropertyUpdater>>> updaters = allUpdaters;
- if (clusterTopology.isNameNodeHAEnabled()) {
+ if (clusterTopology.isNameNodeHAEnabled(configurationContext)) {
updaters = addNameNodeHAUpdaters(updaters);
}
- if (clusterTopology.isYarnResourceManagerHAEnabled()) {
+ if (clusterTopology.isYarnResourceManagerHAEnabled(configurationContext)) {
updaters = addYarnResourceManagerHAUpdaters(updaters);
}
@@ -1060,7 +1050,7 @@ public class BlueprintConfigurationProcessor {
*/
private boolean shouldPropertyBeExcludedForBlueprintExport(String propertyName, String propertyValue, String propertyType, ClusterTopology topology, PropertyFilter [] exportPropertyFilters ) {
for(PropertyFilter filter : exportPropertyFilters) {
- if (!filter.isPropertyIncluded(propertyName, propertyValue, propertyType, topology)) {
+ if (!filter.isPropertyIncluded(propertyName, propertyValue, propertyType, topology, configurationContext)) {
return true;
}
}
@@ -1084,11 +1074,12 @@ public class BlueprintConfigurationProcessor {
private static boolean shouldPropertyBeExcludedForClusterUpdate(String propertyName,
String propertyValue,
String propertyType,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
for(PropertyFilter filter : clusterUpdatePropertyFilters) {
try {
- if (!filter.isPropertyIncluded(propertyName, propertyValue, propertyType, topology)) {
+ if (!filter.isPropertyIncluded(propertyName, propertyValue, propertyType, topology, configurationContext)) {
if (!shouldPropertyBeStoredWithDefault(propertyName)) {
return true;
}
@@ -1326,7 +1317,8 @@ public class BlueprintConfigurationProcessor {
Map<String, String> typeProperties = properties.get(type);
if (typeProperties != null && typeProperties.containsKey(propertyName)) {
- String newValue = npu.updateForBlueprintExport(propertyName, typeProperties.get(propertyName), properties, clusterTopology);
+ String newValue = npu.updateForBlueprintExport(propertyName, typeProperties.get(propertyName),
+ properties, clusterTopology, configurationContext);
configuration.setProperty(type, propertyName, newValue);
}
}
@@ -1350,7 +1342,8 @@ public class BlueprintConfigurationProcessor {
String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology);
+ ClusterTopology topology,
+ ConfigurationContext configurationContext);
/**
* Determine the required host groups for the provided property.
@@ -1365,7 +1358,8 @@ public class BlueprintConfigurationProcessor {
Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology);
+ ClusterTopology topology,
+ ConfigurationContext configurationContext);
}
private static class HostGroupUpdater implements PropertyUpdater {
@@ -1376,7 +1370,9 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext
+ ) {
//todo: getHostStrings
Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(origValue);
@@ -1402,7 +1398,7 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology, ConfigurationContext configurationContext) {
//todo: getHostStrings
Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(origValue);
if (m.find()) {
@@ -1446,9 +1442,10 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
- String replacedValue = super.updateForClusterCreate(propertyName, origValue, properties, topology);
+ String replacedValue = super.updateForClusterCreate(propertyName, origValue, properties, topology, configurationContext);
if (!Objects.equals(origValue, replacedValue)) {
return replacedValue;
} else {
@@ -1459,7 +1456,8 @@ public class BlueprintConfigurationProcessor {
topology.getHostAssignmentsForComponent(component).iterator().next(), properties);
} else {
//todo: extract all hard coded HA logic
- Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component);
+
+ Cardinality cardinality = configurationContext.getStack().getCardinality(component);
// if no matching host groups are found for a component whose configuration
// is handled by this updater, check the stack first to determine if
// zero is a valid cardinality for this component. This is necessary
@@ -1468,7 +1466,7 @@ public class BlueprintConfigurationProcessor {
if (matchingGroupCount == 0 && cardinality.isValidCount(0)) {
return origValue;
} else {
- if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroupCount == 2)) {
+ if (topology.isNameNodeHAEnabled(configurationContext) && isComponentNameNode() && (matchingGroupCount == 2)) {
// if this is the defaultFS property, it should reflect the nameservice name,
// rather than a hostname (used in non-HA scenarios)
if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) {
@@ -1494,13 +1492,13 @@ public class BlueprintConfigurationProcessor {
}
- if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroupCount == 0)) {
+ if (topology.isNameNodeHAEnabled(configurationContext) && isComponentSecondaryNameNode() && (matchingGroupCount == 0)) {
// if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE
// eventually this type of information should be encoded in the stacks
return origValue;
}
- if (topology.isYarnResourceManagerHAEnabled() && isComponentResourceManager() && (matchingGroupCount == 2)) {
+ if (topology.isYarnResourceManagerHAEnabled(configurationContext) && isComponentResourceManager() && (matchingGroupCount == 2)) {
if (!origValue.contains("localhost")) {
// if this Yarn property is a FQDN, then simply return it
return origValue;
@@ -1558,8 +1556,10 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
- Collection<String> result = super.getRequiredHostGroups(propertyName, origValue, properties, topology);
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
+ Collection<String> result = super.getRequiredHostGroups(propertyName,
+ origValue, properties, topology, configurationContext);
if (!result.isEmpty()) {
return result;
} else {
@@ -1568,7 +1568,7 @@ public class BlueprintConfigurationProcessor {
if (matchingGroupCount != 0) {
return new HashSet<>(matchingGroups);
} else {
- Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component);
+ Cardinality cardinality = configurationContext.getStack().getCardinality(component);
// if no matching host groups are found for a component whose configuration
// is handled by this updater, return an empty set
if (! cardinality.isValidCount(0)) {
@@ -1715,9 +1715,10 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
try {
- return super.updateForClusterCreate(propertyName, origValue, properties, topology);
+ return super.updateForClusterCreate(propertyName, origValue, properties, topology, configurationContext);
} catch (IllegalArgumentException illegalArgumentException) {
// return the original value, since the optional component is not available in this cluster
return origValue;
@@ -1728,10 +1729,11 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
try {
- return super.getRequiredHostGroups(propertyName, origValue, properties, topology);
+ return super.getRequiredHostGroups(propertyName, origValue, properties, topology, configurationContext);
} catch (IllegalArgumentException e) {
return Collections.emptySet();
}
@@ -1786,10 +1788,11 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
if (isDatabaseManaged(properties)) {
- return super.updateForClusterCreate(propertyName, origValue, properties, topology);
+ return super.updateForClusterCreate(propertyName, origValue, properties, topology, configurationContext);
} else {
return origValue;
}
@@ -1799,9 +1802,10 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
if (isDatabaseManaged(properties)) {
- return super.getRequiredHostGroups(propertyName, origValue, properties, topology);
+ return super.getRequiredHostGroups(propertyName, origValue, properties, topology, configurationContext);
} else {
return Collections.emptySet();
}
@@ -1886,7 +1890,8 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
StringBuilder sb = new StringBuilder();
@@ -2053,7 +2058,8 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
Collection<String> requiredHostGroups = new HashSet<>();
@@ -2108,14 +2114,15 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
// return customer-supplied properties without updating them
if (isFQDNValue(origValue)) {
return origValue;
}
- return doFormat(propertyUpdater.updateForClusterCreate(propertyName, origValue, properties, topology));
+ return doFormat(propertyUpdater.updateForClusterCreate(propertyName, origValue, properties, topology, configurationContext));
}
/**
@@ -2131,9 +2138,10 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
- return propertyUpdater.getRequiredHostGroups(propertyName, origValue, properties, topology);
+ return propertyUpdater.getRequiredHostGroups(propertyName, origValue, properties, topology, configurationContext);
}
/**
@@ -2245,7 +2253,8 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
// always return the original value, since these properties do not require update handling
return origValue;
}
@@ -2254,7 +2263,8 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
return Collections.emptySet();
}
@@ -2284,7 +2294,8 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
// short-circuit out any custom property values defined by the deployer
if (!origValue.contains("%HOSTGROUP") &&
@@ -2309,7 +2320,7 @@ public class BlueprintConfigurationProcessor {
String key = keyValuePair.split("=")[0].trim();
if (mapOfKeysToUpdaters.containsKey(key)) {
String result = mapOfKeysToUpdaters.get(key).updateForClusterCreate(
- key, keyValuePair.split("=")[1].trim(), properties, topology);
+ key, keyValuePair.split("=")[1].trim(), properties, topology, configurationContext);
// append the internal property result, escape out any commas in the internal property,
// this is required due to the specific syntax of templeton.hive.properties
updatedResult.append(key);
@@ -2327,7 +2338,8 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
// short-circuit out any custom property values defined by the deployer
if (!origValue.contains("%HOSTGROUP") &&
@@ -2344,7 +2356,7 @@ public class BlueprintConfigurationProcessor {
String key = keyValuePair.split("=")[0];
if (mapOfKeysToUpdaters.containsKey(key)) {
requiredGroups.addAll(mapOfKeysToUpdaters.get(key).getRequiredHostGroups(
- propertyName, keyValuePair.split("=")[1], properties, topology));
+ propertyName, keyValuePair.split("=")[1], properties, topology, configurationContext));
}
}
return requiredGroups;
@@ -2360,14 +2372,16 @@ public class BlueprintConfigurationProcessor {
public Collection<String> getRequiredHostGroups(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
return Collections.emptyList();
}
public String updateForBlueprintExport(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
return origValue;
}
}
@@ -2570,7 +2584,8 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
String atlasHookClass = "org.apache.atlas.hive.hook.HiveHook";
String[] hiveHooks = origValue.split(",");
@@ -2581,7 +2596,7 @@ public class BlueprintConfigurationProcessor {
}
}
- boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS");
+ boolean isAtlasInCluster = topology.getBlueprint().getAllServiceTypes().contains("ATLAS");
boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook"));
// Append atlas hook if not already present.
@@ -2610,9 +2625,10 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
- if (topology.getBlueprint().getServices().contains("ATLAS")) {
+ if (topology.getBlueprint().getAllServiceTypes().contains("ATLAS")) {
// if original value is not set or is the default "primary" set the cluster id
if (origValue == null || origValue.trim().isEmpty() || origValue.equals("primary")) {
//use cluster id because cluster name may change
@@ -2630,7 +2646,7 @@ public class BlueprintConfigurationProcessor {
public String updateForBlueprintExport(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology, ConfigurationContext configurationContext) {
// if the value is the cluster id, then update to primary
if (origValue.equals(String.valueOf(topology.getClusterId()))) {
@@ -2646,8 +2662,9 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
- if (topology.getBlueprint().getServices().contains("ATLAS")) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
+ if (topology.getBlueprint().getAllServiceTypes().contains("ATLAS")) {
String host = topology.getHostAssignmentsForComponent("ATLAS_SERVER").iterator().next();
boolean tlsEnabled = Boolean.parseBoolean(properties.get("application-properties").get("atlas.enableTLS"));
@@ -2706,9 +2723,10 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
- if (topology.getBlueprint().getServices().contains("AMBARI_METRICS")) {
+ if (topology.getBlueprint().getAllServiceTypes().contains("AMBARI_METRICS")) {
final String amsReporterClass = "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter";
if (origValue == null || origValue.isEmpty()) {
return amsReporterClass;
@@ -2739,9 +2757,10 @@ public class BlueprintConfigurationProcessor {
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
- if (topology.getBlueprint().getServices().contains("AMBARI_METRICS")) {
+ if (topology.getBlueprint().getAllServiceTypes().contains("AMBARI_METRICS")) {
final String amsReportesClass = "org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter";
if (origValue == null || origValue.isEmpty()) {
return amsReportesClass;
@@ -2801,7 +2820,8 @@ public class BlueprintConfigurationProcessor {
// AMS
amsSiteMap.put("timeline.metrics.service.webapp.address", new SingleHostTopologyUpdater("METRICS_COLLECTOR") {
@Override
- public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) {
+ public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
if (!origValue.startsWith(BIND_ALL_IP_ADDRESS)) {
return origValue.replace(origValue.split(":")[0], BIND_ALL_IP_ADDRESS);
} else {
@@ -2833,7 +2853,7 @@ public class BlueprintConfigurationProcessor {
// AMBARI-5206
final Map<String , String> userProps = new HashMap<>();
- Collection<String> services = clusterTopology.getBlueprint().getServices();
+ Collection<String> services = clusterTopology.getBlueprint().getAllServiceTypes();
if (services.contains("HDFS")) {
// only add user properties to the map for
// services actually included in the blueprint definition
@@ -2884,17 +2904,17 @@ public class BlueprintConfigurationProcessor {
* In case the excluded config-type related service is not present in the blueprint, excluded configs are ignored
* @param configuration
* @param configTypesUpdated
- * @param stack
*/
- private void addExcludedConfigProperties(Configuration configuration, Set<String> configTypesUpdated, Stack stack) {
- Collection<String> blueprintServices = clusterTopology.getBlueprint().getServices();
+ private void addExcludedConfigProperties(Configuration configuration, Set<String> configTypesUpdated) {
+ Collection<Service> blueprintServices = clusterTopology.getBlueprint().getAllServices();
LOG.debug("Handling excluded properties for blueprint services: {}", blueprintServices);
- for (String blueprintService : blueprintServices) {
+ for (Service blueprintService : blueprintServices) {
LOG.debug("Handling excluded properties for blueprint service: {}", blueprintService);
- Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(blueprintService);
+ StackV2 stack = blueprintService.getStack();
+ Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(blueprintService.getType());
if (excludedConfigTypes.isEmpty()) {
LOG.debug("There are no excluded config types for blueprint service: {}", blueprintService);
@@ -2924,7 +2944,7 @@ public class BlueprintConfigurationProcessor {
continue;
}
- Map<String, String> configProperties = stack.getConfigurationProperties(blueprintService, configType);
+ Map<String, String> configProperties = stack.getConfigurationProperties(blueprintService.getType(), configType);
for(Map.Entry<String, String> entry: configProperties.entrySet()) {
LOG.debug("ADD property {} {} {}", configType, entry.getKey(), entry.getValue());
ensureProperty(configuration, configType, entry.getKey(), entry.getValue(), configTypesUpdated);
@@ -2982,9 +3002,8 @@ public class BlueprintConfigurationProcessor {
private void setStackToolsAndFeatures(Configuration configuration, Set<String> configTypesUpdated)
throws ConfigurationTopologyException {
ConfigHelper configHelper = clusterTopology.getAmbariContext().getConfigHelper();
- Stack stack = clusterTopology.getBlueprint().getStack();
- String stackName = stack.getName();
- String stackVersion = stack.getVersion();
+ String stackName = configurationContext.getStack().getName();
+ String stackVersion = configurationContext.getStack().getVersion();
StackId stackId = new StackId(stackName, stackVersion);
@@ -3047,7 +3066,8 @@ public class BlueprintConfigurationProcessor {
* @return true if the property should be included
* false if the property should not be included
*/
- boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology);
+ boolean isPropertyIncluded(String propertyName, String propertyValue,
+ String configType, ClusterTopology topology, ConfigurationContext configurationContext);
}
/**
@@ -3081,7 +3101,8 @@ public class BlueprintConfigurationProcessor {
* false if the property should not be included
*/
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
return !PASSWORD_NAME_REGEX.matcher(propertyName).matches();
}
}
@@ -3106,8 +3127,9 @@ public class BlueprintConfigurationProcessor {
* false if the property should not be included
*/
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
- Stack stack = topology.getBlueprint().getStack();
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
+ StackV2 stack = configurationContext.getStack();
final String serviceName = stack.getServiceForConfigType(configType);
return !(stack.isPasswordProperty(serviceName, configType, propertyName) ||
stack.isKerberosPrincipalNameProperty(serviceName, configType, propertyName));
@@ -3138,8 +3160,10 @@ public class BlueprintConfigurationProcessor {
this.authToLocalPerClusterMap = authToLocalPerClusterMap;
}
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
- return (authToLocalPerClusterMap == null || authToLocalPerClusterMap.get(topology.getClusterId()) == null || !authToLocalPerClusterMap.get(topology.getClusterId()).contains(String.format("%s/%s", configType, propertyName)));
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
+ return (authToLocalPerClusterMap == null || authToLocalPerClusterMap.get(topology.getClusterId()) == null ||
+ !authToLocalPerClusterMap.get(topology.getClusterId()).contains(String.format("%s/%s", configType, propertyName)));
}
}
@@ -3161,7 +3185,8 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
return !(propertyConfigType.equals(configType) &&
this.propertyName.equals(propertyName));
}
@@ -3203,15 +3228,15 @@ public class BlueprintConfigurationProcessor {
* false if the property should not be included
*/
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
- Stack stack = topology.getBlueprint().getStack();
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
Configuration configuration = topology.getConfiguration();
- final String serviceName = stack.getServiceForConfigType(configType);
- Map<String, Stack.ConfigProperty> typeProperties =
- stack.getConfigurationPropertiesWithMetadata(serviceName, configType);
+ final String serviceName = configurationContext.getStack().getServiceForConfigType(configType);
+ Map<String, StackV2.ConfigProperty> typeProperties =
+ configurationContext.getStack().getConfigurationPropertiesWithMetadata(serviceName, configType);
- Stack.ConfigProperty configProperty = typeProperties.get(propertyName);
+ StackV2.ConfigProperty configProperty = typeProperties.get(propertyName);
if (configProperty != null) {
Set<PropertyDependencyInfo> dependencyInfos = configProperty.getDependsOnProperties();
if (dependencyInfos != null) {
@@ -3332,8 +3357,9 @@ public class BlueprintConfigurationProcessor {
* false if the property should not be included
*/
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
- if (topology.isNameNodeHAEnabled()) {
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
+ if (topology.isNameNodeHAEnabled(configurationContext)) {
if (setOfHDFSPropertyNamesNonHA.contains(propertyName)) {
return false;
}
@@ -3369,7 +3395,8 @@ public class BlueprintConfigurationProcessor {
* false if the property should not be included
*/
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
if (configType.equals(this.configType) && propertyName.equals(this.propertyName) && propertyValue.equals(this
.propertyValue)) {
return false;
@@ -3402,7 +3429,8 @@ public class BlueprintConfigurationProcessor {
* false if the property should not be included
*/
@Override
- public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
+ public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
int matchingGroupCount = topology.getHostGroupsForComponent(HAWQSTANDBY).size();
if (matchingGroupCount == 0) {
if (setOfHawqPropertyNamesNonHA.contains(propertyName)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigurationContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigurationContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigurationContext.java
new file mode 100644
index 0000000..983bb9f
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ConfigurationContext.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import org.apache.ambari.server.topology.Configuration;
+
+import java.util.Map;
+
+/**
+ * Provides a context for configuration.
+ */
+public class ConfigurationContext {
+
+ private final Configuration configuration;
+
+ private final StackV2 stack;
+
+ public ConfigurationContext(StackV2 stack, Configuration configuration){
+ this.stack = stack;
+ this.configuration = configuration;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public StackV2 getStack() {
+ return stack;
+ }
+
+ public boolean isNameNodeHAEnabled() {
+ Map<String, Map<String, String>> configurationProperties = getConfiguration().getProperties();
+ return configurationProperties.containsKey("hdfs-site") &&
+ (configurationProperties.get("hdfs-site").containsKey("dfs.nameservices") ||
+ configurationProperties.get("hdfs-site").containsKey("dfs.internal.nameservices"));
+ }
+
+ public boolean isYarnResourceManagerHAEnabled() {
+ Map<String, Map<String, String>> configProperties = getConfiguration().getProperties();
+ return configProperties.containsKey("yarn-site") && configProperties.get("yarn-site").containsKey("yarn.resourcemanager.ha.enabled")
+ && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
index 16d3114..800e2ff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
@@ -19,17 +19,6 @@
package org.apache.ambari.server.controller.internal;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.util.TreeNode;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -38,18 +27,14 @@ import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.HostConfig;
-import org.apache.ambari.server.topology.Blueprint;
-import org.apache.ambari.server.topology.BlueprintImpl;
-import org.apache.ambari.server.topology.Component;
-import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.HostGroupImpl;
-import org.apache.ambari.server.topology.HostGroupInfo;
-import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
-import org.apache.ambari.server.topology.TopologyRequest;
+import org.apache.ambari.server.topology.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+
/**
* Request to export a blueprint from an existing cluster.
*/
@@ -60,8 +45,12 @@ public class ExportBlueprintRequest implements TopologyRequest {
private String clusterName;
private Long clusterId;
- private Blueprint blueprint;
- private Configuration configuration;
+ private BlueprintV2 blueprint;
+ /**
+ * List of services
+ */
+ protected Collection<Service> services;
+
//todo: Should this map be represented by a new class?
private Map<String, HostGroupInfo> hostGroupInfo = new HashMap<>();
@@ -98,13 +87,18 @@ public class ExportBlueprintRequest implements TopologyRequest {
}
@Override
- public Blueprint getBlueprint() {
+ public BlueprintV2 getBlueprint() {
return blueprint;
}
@Override
+ public Collection<Service> getServiceConfigs() {
+ return services;
+ }
+
+ @Override
public Configuration getConfiguration() {
- return configuration;
+ return null;
}
@Override
@@ -135,7 +129,7 @@ public class ExportBlueprintRequest implements TopologyRequest {
hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, stack, componentList,
exportedHostGroup.getConfiguration(), String.valueOf(exportedHostGroup.getCardinality())));
}
- blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null);
+ //blueprint = new BlueprintImplV2(bpName, hostGroups, stack, configuration, null);
}
private void createHostGroupInfo(Collection<ExportedHostGroup> exportedHostGroups) {
@@ -183,11 +177,12 @@ public class ExportBlueprintRequest implements TopologyRequest {
attributes.put(configuration.getType(), configuration.getPropertyAttributes());
}
}
- configuration = new Configuration(properties, attributes);
- // empty parent configuration when exporting as all properties are included in this configuration
- configuration.setParentConfiguration(new Configuration(
- Collections.emptyMap(),
- Collections.emptyMap()));
+// configuration = new Configuration(properties, attributes);
+// // empty parent configuration when exporting as all properties are included in this configuration
+// configuration.setParentConfiguration(new Configuration(
+// Collections.emptyMap(),
+// Collections.emptyMap()));
+
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
index 1fd6091..6d72b21 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
@@ -17,31 +17,19 @@
*/
package org.apache.ambari.server.controller.internal;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.base.Enums;
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
import org.apache.ambari.server.api.predicate.InvalidQueryException;
import org.apache.ambari.server.security.encryption.CredentialStoreType;
import org.apache.ambari.server.stack.NoSuchStackException;
import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileBuilder;
import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileEvaluationException;
-import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
-import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.ConfigurationFactory;
-import org.apache.ambari.server.topology.Credential;
-import org.apache.ambari.server.topology.HostGroupInfo;
-import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
-import org.apache.ambari.server.topology.NoSuchBlueprintException;
-import org.apache.ambari.server.topology.SecurityConfiguration;
+import org.apache.ambari.server.topology.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Enums;
-import com.google.common.base.Optional;
-import com.google.common.base.Strings;
+import java.util.*;
/**
* Request for provisioning a cluster.
@@ -183,10 +171,14 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
this.securityConfiguration = securityConfiguration;
- Configuration configuration = configurationFactory.getConfiguration(
- (Collection<Map<String, String>>) properties.get(CONFIGURATIONS_PROPERTY));
- configuration.setParentConfiguration(blueprint.getConfiguration());
- setConfiguration(configuration);
+// Configuration configuration = configurationFactory.getConfiguration(
+// (Collection<Map<String, String>>) properties.get(CONFIGURATIONS_PROPERTY));
+// configuration.setParentConfiguration(blueprint.getConfiguration());
+// setConfiguration(configuration);
+
+ //TODO load services, merge servie configs from Cluster template with service configs from Blueprint
+ serviceConfigs = new ArrayList<>();
+
parseHostGroupInfo(properties);
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ScaleClusterRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ScaleClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ScaleClusterRequest.java
index fe33f93..314fb26 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ScaleClusterRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ScaleClusterRequest.java
@@ -19,19 +19,19 @@
package org.apache.ambari.server.controller.internal;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ambari.server.api.predicate.InvalidQueryException;
import org.apache.ambari.server.stack.NoSuchStackException;
-import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.BlueprintV2;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.HostGroupInfo;
import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
/**
* A request for a scaling an existing cluster.
*/
@@ -58,8 +58,8 @@ public class ScaleClusterRequest extends BaseClusterRequest {
setClusterName(String.valueOf(properties.get(HostResourceProvider.HOST_CLUSTER_NAME_PROPERTY_ID)));
}
// currently don't allow cluster scoped configuration in scaling operation
- setConfiguration(new Configuration(Collections.emptyMap(),
- Collections.emptyMap()));
+// setConfiguration(new Configuration(Collections.emptyMap(),
+// Collections.emptyMap()));
parseHostGroups(properties);
}
@@ -110,7 +110,7 @@ public class ScaleClusterRequest extends BaseClusterRequest {
throw new InvalidTopologyTemplateException("A name must be specified for all host groups");
}
- Blueprint blueprint = getBlueprint();
+ BlueprintV2 blueprint = getBlueprint();
if (getBlueprint() == null) {
blueprint = parseBlueprint(blueprintName);
setBlueprint(blueprint);
@@ -197,8 +197,8 @@ public class ScaleClusterRequest extends BaseClusterRequest {
*
* @throws InvalidTopologyTemplateException if specified blueprint or stack doesn't exist
*/
- private Blueprint parseBlueprint(String blueprintName) throws InvalidTopologyTemplateException {
- Blueprint blueprint;
+ private BlueprintV2 parseBlueprint(String blueprintName) throws InvalidTopologyTemplateException {
+ BlueprintV2 blueprint;
try {
blueprint = getBlueprintFactory().getBlueprint(blueprintName);
} catch (NoSuchStackException e) {
[4/4] ambari git commit: use V2 objects in TopologyManager
Posted by ma...@apache.org.
use V2 objects in TopologyManager
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/65d44cd5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/65d44cd5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/65d44cd5
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 65d44cd505f5056499dc29f91d911e604a0b72d8
Parents: b0ff5da
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Mon Oct 16 16:27:50 2017 +0200
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Mon Oct 16 20:14:17 2017 +0200
----------------------------------------------------------------------
.../query/render/ClusterBlueprintRenderer.java | 60 +-
.../server/api/services/ServiceService.java | 2 +-
.../StackAdvisorBlueprintProcessor.java | 37 +-
.../users/ActiveWidgetLayoutService.java | 2 +-
.../api/services/views/ViewInstanceService.java | 2 +-
.../ambari/server/controller/StackV2.java | 826 ------------------
.../controller/internal/BaseClusterRequest.java | 41 +-
.../BlueprintConfigurationProcessor.java | 304 ++++---
.../internal/ConfigurationContext.java | 59 ++
.../internal/ExportBlueprintRequest.java | 55 +-
.../internal/ProvisionClusterRequest.java | 34 +-
.../internal/ScaleClusterRequest.java | 20 +-
.../server/controller/internal/StackV2.java | 829 +++++++++++++++++++
.../server/controller/internal/UnitUpdater.java | 20 +-
.../ambari/server/topology/AmbariContext.java | 297 +++----
.../server/topology/BlueprintFactory.java | 14 +-
.../ambari/server/topology/BlueprintImplV2.java | 40 +-
.../ambari/server/topology/BlueprintV2.java | 42 +-
.../server/topology/BlueprintValidatorImpl.java | 78 +-
.../topology/ClusterConfigurationRequest.java | 79 +-
.../ambari/server/topology/ClusterTopology.java | 21 +-
.../server/topology/ClusterTopologyImpl.java | 93 +--
.../ambari/server/topology/ComponentV2.java | 17 +-
.../ambari/server/topology/HostGroupInfo.java | 27 +-
.../ambari/server/topology/HostGroupV2.java | 6 +
.../ambari/server/topology/HostRequest.java | 58 +-
.../ambari/server/topology/LogicalRequest.java | 26 +-
.../server/topology/PersistedStateImpl.java | 55 +-
.../apache/ambari/server/topology/Service.java | 25 +-
.../ambari/server/topology/ServiceGroup.java | 1 +
.../ambari/server/topology/TopologyManager.java | 62 +-
.../ambari/server/topology/TopologyRequest.java | 10 +-
.../tasks/PersistHostResourcesTask.java | 23 +-
.../validators/ClusterConfigTypeValidator.java | 11 +-
.../validators/HiveServiceValidator.java | 50 +-
.../RequiredConfigPropertiesValidator.java | 102 +--
.../validators/RequiredPasswordValidator.java | 32 +-
.../validators/StackConfigTypeValidator.java | 37 +-
.../topology/validators/UnitValidator.java | 42 +-
.../server/api/services/AmbariMetaInfoTest.java | 2 +-
.../api/services/RootServiceServiceTest.java | 2 +-
.../server/api/services/ServiceServiceTest.java | 2 +-
.../AmbariManagementControllerTest.java | 6 +-
.../server/state/cluster/ClusterTest.java | 2 +-
.../server/topology/BlueprintImplTest.java | 10 +-
.../topology/ClusterTopologyImplTest.java | 58 +-
46 files changed, 1782 insertions(+), 1839 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index acdf9ed..4f71255 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -18,57 +18,24 @@
package org.apache.ambari.server.api.query.render;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.query.QueryInfo;
import org.apache.ambari.server.api.services.Request;
-import org.apache.ambari.server.api.services.Result;
-import org.apache.ambari.server.api.services.ResultImpl;
-import org.apache.ambari.server.api.services.ResultPostProcessor;
-import org.apache.ambari.server.api.services.ResultPostProcessorImpl;
+import org.apache.ambari.server.api.services.*;
import org.apache.ambari.server.api.util.TreeNode;
import org.apache.ambari.server.api.util.TreeNodeImpl;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
-import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
-import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
-import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
-import org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
-import org.apache.ambari.server.controller.internal.RequestImpl;
-import org.apache.ambari.server.controller.internal.ResourceImpl;
-import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.controller.spi.ClusterController;
-import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
-import org.apache.ambari.server.controller.spi.NoSuchResourceException;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.spi.ResourceProvider;
-import org.apache.ambari.server.controller.spi.SystemException;
-import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
+import org.apache.ambari.server.controller.internal.*;
+import org.apache.ambari.server.controller.spi.*;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.apache.ambari.server.state.SecurityType;
-import org.apache.ambari.server.topology.AmbariContext;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.ClusterTopologyImpl;
-import org.apache.ambari.server.topology.Component;
-import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.HostGroupInfo;
-import org.apache.ambari.server.topology.InvalidTopologyException;
-import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
-import org.apache.ambari.server.topology.SecurityConfigurationFactory;
+import org.apache.ambari.server.topology.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+
/**
* Renderer which renders a cluster resource as a blueprint.
*/
@@ -193,12 +160,15 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
throw new RuntimeException("Unable to process blueprint export request: " + e, e);
}
- BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+ ConfigurationContext configurationContext = new ConfigurationContext(topology.getBlueprint().getStacks().iterator().next(),
+ topology.getBlueprint().getConfiguration());
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology, configurationContext);
configProcessor.doUpdateForBlueprintExport();
- Stack stack = topology.getBlueprint().getStack();
- blueprintResource.setProperty("Blueprints/stack_name", stack.getName());
- blueprintResource.setProperty("Blueprints/stack_version", stack.getVersion());
+ //TODO add service groups
+ //Stack stack = topology.getBlueprint().getStack();
+ //blueprintResource.setProperty("Blueprints/stack_name", stack.getName());
+ //blueprintResource.setProperty("Blueprints/stack_version", stack.getVersion());
if (topology.isClusterKerberosEnabled()) {
Map<String, Object> securityConfigMap = new LinkedHashMap<>();
@@ -443,9 +413,9 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
*
* @return list of component names for the host
*/
- private List<Map<String, String>> processHostGroupComponents(HostGroup group) {
+ private List<Map<String, String>> processHostGroupComponents(HostGroupV2 group) {
List<Map<String, String>> listHostGroupComponents = new ArrayList<>();
- for (Component component : group.getComponents()) {
+ for (ComponentV2 component : group.getComponents()) {
Map<String, String> mapComponentProperties = new HashMap<>();
listHostGroupComponents.add(mapComponentProperties);
mapComponentProperties.put("name", component.getName());
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
index 12f4bca..cb0ad20 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
@@ -122,7 +122,7 @@ public class ServiceService extends BaseService {
@Path("") // This is needed if class level path is not present otherwise no Swagger docs will be generated for this method
@Produces(MediaType.TEXT_PLAIN)
@ApiOperation(value = "Get all services",
- nickname = "ServiceService#getServices",
+ nickname = "ServiceService#getServiceConfigs",
notes = "Returns all services.",
response = ServiceResponse.ServiceResponseSwagger.class,
responseContainer = RESPONSE_CONTAINER_LIST)
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
index 273c0ff..441f71d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
@@ -18,34 +18,23 @@
package org.apache.ambari.server.api.services.stackadvisor;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.inject.Singleton;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.BlueprintConfigurations;
import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
-import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.internal.StackV2;
import org.apache.ambari.server.state.ValueAttributesInfo;
-import org.apache.ambari.server.topology.AdvisedConfiguration;
-import org.apache.ambari.server.topology.Blueprint;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
-import org.apache.ambari.server.topology.HostGroup;
-import org.apache.ambari.server.topology.HostGroupInfo;
+import org.apache.ambari.server.topology.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.inject.Singleton;
+import java.util.*;
/**
* Generate advised configurations for blueprint cluster provisioning by the stack advisor.
@@ -89,14 +78,14 @@ public class StackAdvisorBlueprintProcessor {
}
private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTopology, StackAdvisorRequestType requestType) {
- Stack stack = clusterTopology.getBlueprint().getStack();
+ StackV2 stack = clusterTopology.getBlueprint().getStacks().iterator().next();
Map<String, Set<String>> hgComponentsMap = gatherHostGroupComponents(clusterTopology);
Map<String, Set<String>> hgHostsMap = gatherHostGroupBindings(clusterTopology);
Map<String, Set<String>> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap,
hgHostsMap);
return StackAdvisorRequest.StackAdvisorRequestBuilder
.forStack(stack.getName(), stack.getVersion())
- .forServices(new ArrayList<>(clusterTopology.getBlueprint().getServices()))
+ .forServices(new ArrayList<>(clusterTopology.getBlueprint().getAllServiceTypes()))
.forHosts(gatherHosts(clusterTopology))
.forHostsGroupBindings(gatherHostGroupBindings(clusterTopology))
.forHostComponents(gatherHostGroupComponents(clusterTopology))
@@ -117,7 +106,7 @@ public class StackAdvisorBlueprintProcessor {
private Map<String, Set<String>> gatherHostGroupComponents(ClusterTopology clusterTopology) {
Map<String, Set<String>> hgComponentsMap = Maps.newHashMap();
- for (Map.Entry<String, HostGroup> hgEnrty: clusterTopology.getBlueprint().getHostGroups().entrySet()) {
+ for (Map.Entry<String, HostGroupV2> hgEnrty: clusterTopology.getBlueprint().getHostGroups().entrySet()) {
hgComponentsMap.put(hgEnrty.getKey(), Sets.newCopyOnWriteArraySet(hgEnrty.getValue().getComponentNames()));
}
return hgComponentsMap;
@@ -176,7 +165,7 @@ public class StackAdvisorBlueprintProcessor {
Map<String, BlueprintConfigurations> recommendedConfigurations =
response.getRecommendations().getBlueprint().getConfigurations();
- Blueprint blueprint = topology.getBlueprint();
+ BlueprintV2 blueprint = topology.getBlueprint();
for (Map.Entry<String, BlueprintConfigurations> configEntry : recommendedConfigurations.entrySet()) {
String configType = configEntry.getKey();
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/api/services/users/ActiveWidgetLayoutService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/users/ActiveWidgetLayoutService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/users/ActiveWidgetLayoutService.java
index e7cdabb..2667a0c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/users/ActiveWidgetLayoutService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/users/ActiveWidgetLayoutService.java
@@ -64,7 +64,7 @@ public class ActiveWidgetLayoutService extends BaseService {
*/
@GET
@Produces("text/plain")
- @ApiOperation(value = "Get user widget layouts", nickname = "ActiveWidgetLayoutService#getServices", notes = "Returns all active widget layouts for user.", response = ActiveWidgetLayoutResponse.class, responseContainer = "List")
+ @ApiOperation(value = "Get user widget layouts", nickname = "ActiveWidgetLayoutService#getServiceConfigs", notes = "Returns all active widget layouts for user.", response = ActiveWidgetLayoutResponse.class, responseContainer = "List")
@ApiImplicitParams({
@ApiImplicitParam(name = "fields", value = "Filter user layout details", defaultValue = "WidgetLayoutInfo/*", dataType = "string", paramType = "query"),
@ApiImplicitParam(name = "sortBy", value = "Sort layouts (asc | desc)", defaultValue = "WidgetLayoutInfo/user_name.asc", dataType = "string", paramType = "query"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/api/services/views/ViewInstanceService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/views/ViewInstanceService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/views/ViewInstanceService.java
index e4ebedb..036f1bd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/views/ViewInstanceService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/views/ViewInstanceService.java
@@ -76,7 +76,7 @@ public class ViewInstanceService extends BaseService {
*/
@GET
@Produces("text/plain")
- @ApiOperation(value = "Get all view instances", nickname = "ViewInstanceService#getServices", notes = "Returns all instances for a view version.", response = ViewInstanceResponse.class, responseContainer = "List")
+ @ApiOperation(value = "Get all view instances", nickname = "ViewInstanceService#getServiceConfigs", notes = "Returns all instances for a view version.", response = ViewInstanceResponse.class, responseContainer = "List")
@ApiImplicitParams({
@ApiImplicitParam(name = "fields", value = "Filter view instance details", defaultValue = "ViewInstanceInfo/*", dataType = "string", paramType = "query"),
@ApiImplicitParam(name = "sortBy", value = "Sort users (asc | desc)", defaultValue = "ViewInstanceInfo/instance_name.desc", dataType = "string", paramType = "query"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/StackV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackV2.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackV2.java
deleted file mode 100644
index a6d8979..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackV2.java
+++ /dev/null
@@ -1,826 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.controller;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.orm.entities.StackEntity;
-import org.apache.ambari.server.state.AutoDeployInfo;
-import org.apache.ambari.server.state.ComponentInfo;
-import org.apache.ambari.server.state.DependencyInfo;
-import org.apache.ambari.server.state.PropertyDependencyInfo;
-import org.apache.ambari.server.state.PropertyInfo;
-import org.apache.ambari.server.state.ValueAttributesInfo;
-import org.apache.ambari.server.topology.Cardinality;
-import org.apache.ambari.server.topology.Configuration;
-
-/**
- * Encapsulates stack information.
- */
-public class StackV2 {
- /**
- * Stack name
- */
- private String name;
-
- /**
- * Stack version
- */
- private String version;
-
- /**
- * Repo version
- */
- private String repoVersion;
-
- /**
- * Map of service name to components
- */
- private Map<String, Collection<String>> serviceComponents =
- new HashMap<>();
-
- /**
- * Map of component to service
- */
- private Map<String, String> componentService = new HashMap<>();
-
- /**
- * Map of component to dependencies
- */
- private Map<String, Collection<DependencyInfo>> dependencies =
- new HashMap<>();
-
- /**
- * Map of dependency to conditional service
- */
- private Map<DependencyInfo, String> dependencyConditionalServiceMap =
- new HashMap<>();
-
- /**
- * Map of database component name to configuration property which indicates whether
- * the database in to be managed or if it is an external non-managed instance.
- * If the value of the config property starts with 'New', the database is determined
- * to be managed, otherwise it is non-managed.
- */
- private Map<String, String> dbDependencyInfo = new HashMap<>();
-
- /**
- * Map of component to required cardinality
- */
- private Map<String, String> cardinalityRequirements = new HashMap<>();
-
- //todo: instead of all these maps from component -> * ,
- //todo: we should use a Component object with all of these attributes
- private Set<String> masterComponents = new HashSet<>();
-
- /**
- * Map of component to auto-deploy information
- */
- private Map<String, AutoDeployInfo> componentAutoDeployInfo =
- new HashMap<>();
-
- /**
- * Map of service to config type properties
- */
- private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations =
- new HashMap<>();
-
- /**
- * Map of service to required type properties
- */
- private Map<String, Map<String, Map<String, ConfigProperty>>> requiredServiceConfigurations =
- new HashMap<>();
-
- /**
- * Map of service to config type properties
- */
- private Map<String, Map<String, ConfigProperty>> stackConfigurations =
- new HashMap<>();
-
- /**
- * Map of service to set of excluded config types
- */
- private Map<String, Set<String>> excludedConfigurationTypes =
- new HashMap<>();
-
- /**
- * Ambari Management Controller, used to obtain Stack definitions
- */
- private final AmbariManagementController controller;
-
-
- /**
- * Constructor.
- *
- * @param stack
- * the stack (not {@code null}).
- * @param ambariManagementController
- * the management controller (not {@code null}).
- * @throws AmbariException
- */
- public StackV2(StackEntity stack, AmbariManagementController ambariManagementController) throws AmbariException {
- this(stack.getStackName(), stack.getStackVersion(), ambariManagementController);
- }
-
- /**
- * Constructor.
- *
- * @param name stack name
- * @param version stack version
- *
- * @throws AmbariException an exception occurred getting stack information
- * for the specified name and version
- */
- //todo: don't pass management controller in constructor
- public StackV2(String name, String version, AmbariManagementController controller) throws AmbariException {
- this.name = name;
- this.version = version;
- this.controller = controller;
-
- Set<StackServiceResponse> stackServices = controller.getStackServices(
- Collections.singleton(new StackServiceRequest(name, version, null)));
-
- for (StackServiceResponse stackService : stackServices) {
- String serviceName = stackService.getServiceName();
- parseComponents(serviceName);
- parseExcludedConfigurations(stackService);
- parseConfigurations(stackService);
- registerConditionalDependencies();
- }
-
- //todo: already done for each service
- parseStackConfigurations();
- }
-
- /**
- * Obtain stack name.
- *
- * @return stack name
- */
- public String getName() {
- return name;
- }
-
- /**
- * Obtain stack version.
- *
- * @return stack version
- */
- public String getVersion() {
- return version;
- }
-
- /**
- * Obtain repo version.
- * @return
- */
- public String getRepoVersion() { return repoVersion; }
-
- Map<DependencyInfo, String> getDependencyConditionalServiceMap() {
- return dependencyConditionalServiceMap;
- }
-
- /**
- * Get services contained in the stack.
- *
- * @return collection of all services for the stack
- */
- public Collection<String> getServices() {
- return serviceComponents.keySet();
- }
-
- /**
- * Get components contained in the stack for the specified service.
- *
- * @param service service name
- *
- * @return collection of component names for the specified service
- */
- public Collection<String> getComponents(String service) {
- return serviceComponents.get(service);
- }
-
- /**
- * Get all service components
- *
- * @return map of service to associated components
- */
- public Map<String, Collection<String>> getComponents() {
- Map<String, Collection<String>> serviceComponents = new HashMap<>();
- for (String service : getServices()) {
- Collection<String> components = new HashSet<>();
- components.addAll(getComponents(service));
- serviceComponents.put(service, components);
- }
- return serviceComponents;
- }
-
- /**
- * Get info for the specified component.
- *
- * @param component component name
- *
- * @return component information for the requested component
- * or null if the component doesn't exist in the stack
- */
- public ComponentInfo getComponentInfo(String component) {
- ComponentInfo componentInfo = null;
- String service = getServiceForComponent(component);
- if (service != null) {
- try {
- componentInfo = controller.getAmbariMetaInfo().getComponent(
- getName(), getVersion(), service, component);
- } catch (AmbariException e) {
- // just return null if component doesn't exist
- }
- }
- return componentInfo;
- }
-
- /**
- * Get all configuration types, including excluded types for the specified service.
- *
- * @param service service name
- *
- * @return collection of all configuration types for the specified service
- */
- public Collection<String> getAllConfigurationTypes(String service) {
- return serviceConfigurations.get(service).keySet();
- }
-
- /**
- * Get configuration types for the specified service.
- * This doesn't include any service excluded types.
- *
- * @param service service name
- *
- * @return collection of all configuration types for the specified service
- */
- public Collection<String> getConfigurationTypes(String service) {
- Set<String> serviceTypes = new HashSet<>(serviceConfigurations.get(service).keySet());
- serviceTypes.removeAll(getExcludedConfigurationTypes(service));
-
- return serviceTypes;
- }
-
- /**
- * Get the set of excluded configuration types for this service.
- *
- * @param service service name
- *
- * @return Set of names of excluded config types. Will not return null.
- */
- public Set<String> getExcludedConfigurationTypes(String service) {
- return excludedConfigurationTypes.containsKey(service) ?
- excludedConfigurationTypes.get(service) :
- Collections.emptySet();
- }
-
- /**
- * Get config properties for the specified service and configuration type.
- *
- * @param service service name
- * @param type configuration type
- *
- * @return map of property names to values for the specified service and configuration type
- */
- public Map<String, String> getConfigurationProperties(String service, String type) {
- Map<String, String> configMap = new HashMap<>();
- Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type);
- if (configProperties != null) {
- for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
- configMap.put(configProperty.getKey(), configProperty.getValue().getValue());
- }
- }
- return configMap;
- }
-
- public Map<String, ConfigProperty> getConfigurationPropertiesWithMetadata(String service, String type) {
- return serviceConfigurations.get(service).get(type);
- }
-
- /**
- * Get all required config properties for the specified service.
- *
- * @param service service name
- *
- * @return collection of all required properties for the given service
- */
- public Collection<ConfigProperty> getRequiredConfigurationProperties(String service) {
- Collection<ConfigProperty> requiredConfigProperties = new HashSet<>();
- Map<String, Map<String, ConfigProperty>> serviceProperties = requiredServiceConfigurations.get(service);
- if (serviceProperties != null) {
- for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : serviceProperties.entrySet()) {
- requiredConfigProperties.addAll(typePropertiesEntry.getValue().values());
- }
- }
- return requiredConfigProperties;
- }
-
- /**
- * Get required config properties for the specified service which belong to the specified property type.
- *
- * @param service service name
- * @param propertyType property type
- *
- * @return collection of required properties for the given service and property type
- */
- public Collection<ConfigProperty> getRequiredConfigurationProperties(String service, PropertyInfo.PropertyType propertyType) {
- Collection<ConfigProperty> matchingProperties = new HashSet<>();
- Map<String, Map<String, ConfigProperty>> requiredProperties = requiredServiceConfigurations.get(service);
- if (requiredProperties != null) {
- for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : requiredProperties.entrySet()) {
- for (ConfigProperty configProperty : typePropertiesEntry.getValue().values()) {
- if (configProperty.getPropertyTypes().contains(propertyType)) {
- matchingProperties.add(configProperty);
- }
- }
-
- }
- }
- return matchingProperties;
- }
-
- public boolean isPasswordProperty(String service, String type, String propertyName) {
- return (serviceConfigurations.containsKey(service) &&
- serviceConfigurations.get(service).containsKey(type) &&
- serviceConfigurations.get(service).get(type).containsKey(propertyName) &&
- serviceConfigurations.get(service).get(type).get(propertyName).getPropertyTypes().
- contains(PropertyInfo.PropertyType.PASSWORD));
- }
-
- //todo
- public Map<String, String> getStackConfigurationProperties(String type) {
- Map<String, String> configMap = new HashMap<>();
- Map<String, ConfigProperty> configProperties = stackConfigurations.get(type);
- if (configProperties != null) {
- for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
- configMap.put(configProperty.getKey(), configProperty.getValue().getValue());
- }
- }
- return configMap;
- }
-
- public boolean isKerberosPrincipalNameProperty(String service, String type, String propertyName) {
- return (serviceConfigurations.containsKey(service) &&
- serviceConfigurations.get(service).containsKey(type) &&
- serviceConfigurations.get(service).get(type).containsKey(propertyName) &&
- serviceConfigurations.get(service).get(type).get(propertyName).getPropertyTypes().
- contains(PropertyInfo.PropertyType.KERBEROS_PRINCIPAL));
- }
- /**
- * Get config attributes for the specified service and configuration type.
- *
- * @param service service name
- * @param type configuration type
- *
- * @return map of attribute names to map of property names to attribute values
- * for the specified service and configuration type
- */
- public Map<String, Map<String, String>> getConfigurationAttributes(String service, String type) {
- Map<String, Map<String, String>> attributesMap = new HashMap<>();
- Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type);
- if (configProperties != null) {
- for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
- String propertyName = configProperty.getKey();
- Map<String, String> propertyAttributes = configProperty.getValue().getAttributes();
- if (propertyAttributes != null) {
- for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) {
- String attributeName = propertyAttribute.getKey();
- String attributeValue = propertyAttribute.getValue();
- if (attributeValue != null) {
- Map<String, String> attributes = attributesMap.get(attributeName);
- if (attributes == null) {
- attributes = new HashMap<>();
- attributesMap.put(attributeName, attributes);
- }
- attributes.put(propertyName, attributeValue);
- }
- }
- }
- }
- }
- return attributesMap;
- }
-
- //todo:
- public Map<String, Map<String, String>> getStackConfigurationAttributes(String type) {
- Map<String, Map<String, String>> attributesMap = new HashMap<>();
- Map<String, ConfigProperty> configProperties = stackConfigurations.get(type);
- if (configProperties != null) {
- for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
- String propertyName = configProperty.getKey();
- Map<String, String> propertyAttributes = configProperty.getValue().getAttributes();
- if (propertyAttributes != null) {
- for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) {
- String attributeName = propertyAttribute.getKey();
- String attributeValue = propertyAttribute.getValue();
- Map<String, String> attributes = attributesMap.get(attributeName);
- if (attributes == null) {
- attributes = new HashMap<>();
- attributesMap.put(attributeName, attributes);
- }
- attributes.put(propertyName, attributeValue);
- }
- }
- }
- }
- return attributesMap;
- }
-
- /**
- * Get the service for the specified component.
- *
- * @param component component name
- *
- * @return service name that contains tha specified component
- */
- public String getServiceForComponent(String component) {
- return componentService.get(component);
- }
-
- /**
- * Get the names of the services which contains the specified components.
- *
- * @param components collection of components
- *
- * @return collection of services which contain the specified components
- */
- public Collection<String> getServicesForComponents(Collection<String> components) {
- Set<String> services = new HashSet<>();
- for (String component : components) {
- services.add(getServiceForComponent(component));
- }
-
- return services;
- }
-
- /**
- * Obtain the service name which corresponds to the specified configuration.
- *
- * @param config configuration type
- *
- * @return name of service which corresponds to the specified configuration type
- */
- public String getServiceForConfigType(String config) {
- for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) {
- Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue();
- String serviceName = entry.getKey();
- if (typeMap.containsKey(config) && !getExcludedConfigurationTypes(serviceName).contains(config)) {
- return serviceName;
- }
- }
- throw new IllegalArgumentException(
- "Specified configuration type is not associated with any service: " + config);
- }
-
- /**
- * Return the dependencies specified for the given component.
- *
- * @param component component to get dependency information for
- *
- * @return collection of dependency information for the specified component
- */
- //todo: full dependency graph
- public Collection<DependencyInfo> getDependenciesForComponent(String component) {
- return dependencies.containsKey(component) ? dependencies.get(component) :
- Collections.emptySet();
- }
-
- /**
- * Get the service, if any, that a component dependency is conditional on.
- *
- * @param dependency dependency to get conditional service for
- *
- * @return conditional service for provided component or null if dependency
- * is not conditional on a service
- */
- public String getConditionalServiceForDependency(DependencyInfo dependency) {
- return dependencyConditionalServiceMap.get(dependency);
- }
-
- public String getExternalComponentConfig(String component) {
- return dbDependencyInfo.get(component);
- }
-
- /**
- * Obtain the required cardinality for the specified component.
- */
- public Cardinality getCardinality(String component) {
- return new Cardinality(cardinalityRequirements.get(component));
- }
-
- /**
- * Obtain auto-deploy information for the specified component.
- */
- public AutoDeployInfo getAutoDeployInfo(String component) {
- return componentAutoDeployInfo.get(component);
- }
-
- public boolean isMasterComponent(String component) {
- return masterComponents.contains(component);
- }
-
- public Configuration getConfiguration(Collection<String> services) {
- Map<String, Map<String, Map<String, String>>> attributes = new HashMap<>();
- Map<String, Map<String, String>> properties = new HashMap<>();
-
- for (String service : services) {
- Collection<String> serviceConfigTypes = getConfigurationTypes(service);
- for (String type : serviceConfigTypes) {
- Map<String, String> typeProps = properties.get(type);
- if (typeProps == null) {
- typeProps = new HashMap<>();
- properties.put(type, typeProps);
- }
- typeProps.putAll(getConfigurationProperties(service, type));
-
- Map<String, Map<String, String>> stackTypeAttributes = getConfigurationAttributes(service, type);
- if (!stackTypeAttributes.isEmpty()) {
- if (! attributes.containsKey(type)) {
- attributes.put(type, new HashMap<>());
- }
- Map<String, Map<String, String>> typeAttributes = attributes.get(type);
- for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) {
- String attributeName = attribute.getKey();
- Map<String, String> attributeProps = typeAttributes.get(attributeName);
- if (attributeProps == null) {
- attributeProps = new HashMap<>();
- typeAttributes.put(attributeName, attributeProps);
- }
- attributeProps.putAll(attribute.getValue());
- }
- }
- }
- }
- return new Configuration(properties, attributes);
- }
-
- public Configuration getConfiguration() {
- Map<String, Map<String, Map<String, String>>> stackAttributes = new HashMap<>();
- Map<String, Map<String, String>> stackConfigs = new HashMap<>();
-
- for (String service : getServices()) {
- for (String type : getAllConfigurationTypes(service)) {
- Map<String, String> typeProps = stackConfigs.get(type);
- if (typeProps == null) {
- typeProps = new HashMap<>();
- stackConfigs.put(type, typeProps);
- }
- typeProps.putAll(getConfigurationProperties(service, type));
-
- Map<String, Map<String, String>> stackTypeAttributes = getConfigurationAttributes(service, type);
- if (!stackTypeAttributes.isEmpty()) {
- if (! stackAttributes.containsKey(type)) {
- stackAttributes.put(type, new HashMap<>());
- }
- Map<String, Map<String, String>> typeAttrs = stackAttributes.get(type);
- for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) {
- String attributeName = attribute.getKey();
- Map<String, String> attributes = typeAttrs.get(attributeName);
- if (attributes == null) {
- attributes = new HashMap<>();
- typeAttrs.put(attributeName, attributes);
- }
- attributes.putAll(attribute.getValue());
- }
- }
- }
- }
- return new Configuration(stackConfigs, stackAttributes);
- }
-
- /**
- * Parse components for the specified service from the stack definition.
- *
- * @param service service name
- *
- * @throws AmbariException an exception occurred getting components from the stack definition
- */
- private void parseComponents(String service) throws AmbariException{
- Collection<String> componentSet = new HashSet<>();
-
- Set<StackServiceComponentResponse> components = controller.getStackComponents(
- Collections.singleton(new StackServiceComponentRequest(name, version, service, null)));
-
- // stack service components
- for (StackServiceComponentResponse component : components) {
- String componentName = component.getComponentName();
- componentSet.add(componentName);
- componentService.put(componentName, service);
- String cardinality = component.getCardinality();
- if (cardinality != null) {
- cardinalityRequirements.put(componentName, cardinality);
- }
- AutoDeployInfo autoDeploy = component.getAutoDeploy();
- if (autoDeploy != null) {
- componentAutoDeployInfo.put(componentName, autoDeploy);
- }
-
- // populate component dependencies
- //todo: remove usage of AmbariMetaInfo
- Collection<DependencyInfo> componentDependencies = controller.getAmbariMetaInfo().getComponentDependencies(
- name, version, service, componentName);
-
- if (componentDependencies != null && ! componentDependencies.isEmpty()) {
- dependencies.put(componentName, componentDependencies);
- }
- if (component.isMaster()) {
- masterComponents.add(componentName);
- }
- }
- serviceComponents.put(service, componentSet);
- }
-
- /**
- * Parse configurations for the specified service from the stack definition.
- *
- * @param stackService service to parse the stack configuration for
- *
- * @throws AmbariException an exception occurred getting configurations from the stack definition
- */
- private void parseConfigurations(StackServiceResponse stackService) throws AmbariException {
- String service = stackService.getServiceName();
- Map<String, Map<String, ConfigProperty>> mapServiceConfig = new HashMap<>();
- Map<String, Map<String, ConfigProperty>> mapRequiredServiceConfig = new HashMap<>();
-
-
- serviceConfigurations.put(service, mapServiceConfig);
- requiredServiceConfigurations.put(service, mapRequiredServiceConfig);
-
- Set<ReadOnlyConfigurationResponse> serviceConfigs = controller.getStackConfigurations(
- Collections.singleton(new StackConfigurationRequest(name, version, service, null)));
- Set<ReadOnlyConfigurationResponse> stackLevelConfigs = controller.getStackLevelConfigurations(
- Collections.singleton(new StackLevelConfigurationRequest(name, version, null)));
- serviceConfigs.addAll(stackLevelConfigs);
-
- // shouldn't have any required properties in stack level configuration
- for (ReadOnlyConfigurationResponse config : serviceConfigs) {
- ConfigProperty configProperty = new ConfigProperty(config);
- String type = configProperty.getType();
-
- Map<String, ConfigProperty> mapTypeConfig = mapServiceConfig.get(type);
- if (mapTypeConfig == null) {
- mapTypeConfig = new HashMap<>();
- mapServiceConfig.put(type, mapTypeConfig);
- }
-
- mapTypeConfig.put(config.getPropertyName(), configProperty);
- if (config.isRequired()) {
- Map<String, ConfigProperty> requiredTypeConfig = mapRequiredServiceConfig.get(type);
- if (requiredTypeConfig == null) {
- requiredTypeConfig = new HashMap<>();
- mapRequiredServiceConfig.put(type, requiredTypeConfig);
- }
- requiredTypeConfig.put(config.getPropertyName(), configProperty);
- }
- }
-
- // So far we added only config types that have properties defined
- // in stack service definition. Since there might be config types
- // with no properties defined we need to add those separately
- Set<String> configTypes = stackService.getConfigTypes().keySet();
- for (String configType: configTypes) {
- if (!mapServiceConfig.containsKey(configType)) {
- mapServiceConfig.put(configType, Collections.emptyMap());
- }
- }
- }
-
- private void parseStackConfigurations () throws AmbariException {
-
- Set<ReadOnlyConfigurationResponse> stackLevelConfigs = controller.getStackLevelConfigurations(
- Collections.singleton(new StackLevelConfigurationRequest(name, version, null)));
-
- for (ReadOnlyConfigurationResponse config : stackLevelConfigs) {
- ConfigProperty configProperty = new ConfigProperty(config);
- String type = configProperty.getType();
-
- Map<String, ConfigProperty> mapTypeConfig = stackConfigurations.get(type);
- if (mapTypeConfig == null) {
- mapTypeConfig = new HashMap<>();
- stackConfigurations.put(type, mapTypeConfig);
- }
-
- mapTypeConfig.put(config.getPropertyName(),
- configProperty);
- }
- }
-
- /**
- * Obtain the excluded configuration types from the StackServiceResponse
- *
- * @param stackServiceResponse the response object associated with this stack service
- */
- private void parseExcludedConfigurations(StackServiceResponse stackServiceResponse) {
- excludedConfigurationTypes.put(stackServiceResponse.getServiceName(), stackServiceResponse.getExcludedConfigTypes());
- }
-
- /**
- * Register conditional dependencies.
- */
- //todo: This information should be specified in the stack definition.
- void registerConditionalDependencies() {
- dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database");
- }
-
- /**
- * Contains a configuration property's value and attributes.
- */
- public static class ConfigProperty {
- private ValueAttributesInfo propertyValueAttributes = null;
- private String name;
- private String value;
- private Map<String, String> attributes;
- private Set<PropertyInfo.PropertyType> propertyTypes;
- private String type;
- private Set<PropertyDependencyInfo> dependsOnProperties =
- Collections.emptySet();
-
- public ConfigProperty(ReadOnlyConfigurationResponse config) {
- this.name = config.getPropertyName();
- this.value = config.getPropertyValue();
- this.attributes = config.getPropertyAttributes();
- this.propertyTypes = config.getPropertyType();
- this.type = normalizeType(config.getType());
- this.dependsOnProperties = config.getDependsOnProperties();
- this.propertyValueAttributes = config.getPropertyValueAttributes();
- }
-
- public ConfigProperty(String type, String name, String value) {
- this.type = type;
- this.name = name;
- this.value = value;
- }
-
- public String getName() {
- return name;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
- public String getType() {
- return type;
- }
-
- public Set<PropertyInfo.PropertyType> getPropertyTypes() {
- return propertyTypes;
- }
-
- public void setPropertyTypes(Set<PropertyInfo.PropertyType> propertyTypes) {
- this.propertyTypes = propertyTypes;
- }
-
- public Map<String, String> getAttributes() {
- return attributes;
- }
-
- public void setAttributes(Map<String, String> attributes) {
- this.attributes = attributes;
- }
-
- Set<PropertyDependencyInfo> getDependsOnProperties() {
- return this.dependsOnProperties;
- }
-
- private String normalizeType(String type) {
- //strip .xml from type
- if (type.endsWith(".xml")) {
- type = type.substring(0, type.length() - 4);
- }
- return type;
- }
-
- public ValueAttributesInfo getPropertyValueAttributes() {
- return propertyValueAttributes;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
index 77eafeb..c3858ba 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
@@ -18,24 +18,15 @@
package org.apache.ambari.server.controller.internal;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ambari.server.api.predicate.InvalidQueryException;
import org.apache.ambari.server.api.predicate.QueryLexer;
import org.apache.ambari.server.api.predicate.Token;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.ResourceProvider;
import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
-import org.apache.ambari.server.topology.Blueprint;
-import org.apache.ambari.server.topology.BlueprintFactory;
-import org.apache.ambari.server.topology.Configuration;
-import org.apache.ambari.server.topology.HostGroupInfo;
-import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
-import org.apache.ambari.server.topology.SecurityConfiguration;
-import org.apache.ambari.server.topology.TopologyRequest;
+import org.apache.ambari.server.topology.*;
+
+import java.util.*;
/**
* Provides common cluster request functionality.
@@ -62,12 +53,7 @@ public abstract class BaseClusterRequest implements TopologyRequest {
* blueprint
*/
//todo: change interface to only return blueprint name
- protected Blueprint blueprint;
-
- /**
- * configuration
- */
- protected Configuration configuration;
+ protected BlueprintV2 blueprint;
/**
* security configuration
@@ -80,6 +66,11 @@ public abstract class BaseClusterRequest implements TopologyRequest {
protected static BlueprintFactory blueprintFactory;
/**
+ * List of services
+ */
+ protected Collection<Service> serviceConfigs;
+
+ /**
* Lexer used to obtain property names from a predicate string
*/
private static final QueryLexer queryLexer = new QueryLexer();
@@ -104,13 +95,19 @@ public abstract class BaseClusterRequest implements TopologyRequest {
}
@Override
- public Blueprint getBlueprint() {
+ public BlueprintV2 getBlueprint() {
return blueprint;
}
@Override
+ public Collection<Service> getServiceConfigs() {
+ return serviceConfigs;
+ }
+
+ @Override
+ @Deprecated
public Configuration getConfiguration() {
- return configuration;
+ return null;
}
@Override
@@ -155,7 +152,7 @@ public abstract class BaseClusterRequest implements TopologyRequest {
*
* @param blueprint blueprint
*/
- protected void setBlueprint(Blueprint blueprint) {
+ protected void setBlueprint(BlueprintV2 blueprint) {
this.blueprint = blueprint;
}
@@ -164,8 +161,8 @@ public abstract class BaseClusterRequest implements TopologyRequest {
*
* @param configuration configuration
*/
+ @Deprecated
protected void setConfiguration(Configuration configuration) {
- this.configuration = configuration;
}
/**
[2/4] ambari git commit: use V2 objects in TopologyManager
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java
new file mode 100644
index 0000000..c5a6064
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackV2.java
@@ -0,0 +1,829 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.*;
+import org.apache.ambari.server.orm.entities.StackEntity;
+
+import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.DependencyInfo;
+import org.apache.ambari.server.state.PropertyDependencyInfo;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.ValueAttributesInfo;
+
+import org.apache.ambari.server.topology.Cardinality;
+import org.apache.ambari.server.topology.Configuration;
+
+/**
+ * Encapsulates stack information.
+ */
+public class StackV2 {
+ /**
+ * Stack name
+ */
+ private String name;
+
+ /**
+ * Stack version
+ */
+ private String version;
+
+ /**
+ * Repo version
+ */
+ private String repoVersion;
+
+ /**
+ * Map of service name to components
+ */
+ private Map<String, Collection<String>> serviceComponents =
+ new HashMap<>();
+
+ /**
+ * Map of component to service
+ */
+ private Map<String, String> componentService = new HashMap<>();
+
+ /**
+ * Map of component to dependencies
+ */
+ private Map<String, Collection<DependencyInfo>> dependencies =
+ new HashMap<>();
+
+ /**
+ * Map of dependency to conditional service
+ */
+ private Map<DependencyInfo, String> dependencyConditionalServiceMap =
+ new HashMap<>();
+
+ /**
+ * Map of database component name to configuration property which indicates whether
+ * the database in to be managed or if it is an external non-managed instance.
+ * If the value of the config property starts with 'New', the database is determined
+ * to be managed, otherwise it is non-managed.
+ */
+ private Map<String, String> dbDependencyInfo = new HashMap<>();
+
+ /**
+ * Map of component to required cardinality
+ */
+ private Map<String, String> cardinalityRequirements = new HashMap<>();
+
+ //todo: instead of all these maps from component -> * ,
+ //todo: we should use a Component object with all of these attributes
+ private Set<String> masterComponents = new HashSet<>();
+
+ /**
+ * Map of component to auto-deploy information
+ */
+ private Map<String, AutoDeployInfo> componentAutoDeployInfo =
+ new HashMap<>();
+
+ /**
+ * Map of service to config type properties
+ */
+ private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations =
+ new HashMap<>();
+
+ /**
+ * Map of service to required type properties
+ */
+ private Map<String, Map<String, Map<String, ConfigProperty>>> requiredServiceConfigurations =
+ new HashMap<>();
+
+ /**
+ * Map of service to config type properties
+ */
+ private Map<String, Map<String, ConfigProperty>> stackConfigurations =
+ new HashMap<>();
+
+ /**
+ * Map of service to set of excluded config types
+ */
+ private Map<String, Set<String>> excludedConfigurationTypes =
+ new HashMap<>();
+
+ /**
+ * Ambari Management Controller, used to obtain Stack definitions
+ */
+ private final AmbariManagementController controller;
+
+
+ /**
+ * Constructor.
+ *
+ * @param stack
+ * the stack (not {@code null}).
+ * @param ambariManagementController
+ * the management controller (not {@code null}).
+ * @throws AmbariException
+ */
+ public StackV2(StackEntity stack, AmbariManagementController ambariManagementController) throws AmbariException {
+ this(stack.getStackName(), stack.getStackVersion(), ambariManagementController);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param name stack name
+ * @param version stack version
+ *
+ * @throws AmbariException an exception occurred getting stack information
+ * for the specified name and version
+ */
+ //todo: don't pass management controller in constructor
+ public StackV2(String name, String version, AmbariManagementController controller) throws AmbariException {
+ this.name = name;
+ this.version = version;
+ this.controller = controller;
+
+ Set<StackServiceResponse> stackServices = controller.getStackServices(
+ Collections.singleton(new StackServiceRequest(name, version, null)));
+
+ for (StackServiceResponse stackService : stackServices) {
+ String serviceName = stackService.getServiceName();
+ parseComponents(serviceName);
+ parseExcludedConfigurations(stackService);
+ parseConfigurations(stackService);
+ registerConditionalDependencies();
+ }
+
+ //todo: already done for each service
+ parseStackConfigurations();
+ }
+
+ /**
+ * Obtain stack name.
+ *
+ * @return stack name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Obtain stack version.
+ *
+ * @return stack version
+ */
+ public String getVersion() {
+ return version;
+ }
+
+ /**
+ * Obtain repo version.
+ * @return
+ */
+ public String getRepoVersion() { return repoVersion; }
+
+ Map<DependencyInfo, String> getDependencyConditionalServiceMap() {
+ return dependencyConditionalServiceMap;
+ }
+
+ /**
+ * Get services contained in the stack.
+ *
+ * @return collection of all services for the stack
+ */
+ public Collection<String> getServices() {
+ return serviceComponents.keySet();
+ }
+
+ /**
+ * Get components contained in the stack for the specified service.
+ *
+ * @param service service name
+ *
+ * @return collection of component names for the specified service
+ */
+ public Collection<String> getComponents(String service) {
+ return serviceComponents.get(service);
+ }
+
+ /**
+ * Get all service components
+ *
+ * @return map of service to associated components
+ */
+ public Map<String, Collection<String>> getComponents() {
+ Map<String, Collection<String>> serviceComponents = new HashMap<>();
+ for (String service : getServices()) {
+ Collection<String> components = new HashSet<>();
+ components.addAll(getComponents(service));
+ serviceComponents.put(service, components);
+ }
+ return serviceComponents;
+ }
+
+ /**
+ * Get info for the specified component.
+ *
+ * @param component component name
+ *
+ * @return component information for the requested component
+ * or null if the component doesn't exist in the stack
+ */
+ public ComponentInfo getComponentInfo(String component) {
+ ComponentInfo componentInfo = null;
+ String service = getServiceForComponent(component);
+ if (service != null) {
+ try {
+ componentInfo = controller.getAmbariMetaInfo().getComponent(
+ getName(), getVersion(), service, component);
+ } catch (AmbariException e) {
+ // just return null if component doesn't exist
+ }
+ }
+ return componentInfo;
+ }
+
+ /**
+ * Get all configuration types, including excluded types for the specified service.
+ *
+ * @param service service name
+ *
+ * @return collection of all configuration types for the specified service
+ */
+ public Collection<String> getAllConfigurationTypes(String service) {
+ return serviceConfigurations.get(service).keySet();
+ }
+
+ /**
+ * Get configuration types for the specified service.
+ * This doesn't include any service excluded types.
+ *
+ * @param service service name
+ *
+ * @return collection of all configuration types for the specified service
+ */
+ public Collection<String> getConfigurationTypes(String service) {
+ Set<String> serviceTypes = new HashSet<>(serviceConfigurations.get(service).keySet());
+ serviceTypes.removeAll(getExcludedConfigurationTypes(service));
+
+ return serviceTypes;
+ }
+
+ /**
+ * Get the set of excluded configuration types for this service.
+ *
+ * @param service service name
+ *
+ * @return Set of names of excluded config types. Will not return null.
+ */
+ public Set<String> getExcludedConfigurationTypes(String service) {
+ return excludedConfigurationTypes.containsKey(service) ?
+ excludedConfigurationTypes.get(service) :
+ Collections.emptySet();
+ }
+
+ /**
+ * Get config properties for the specified service and configuration type.
+ *
+ * @param service service name
+ * @param type configuration type
+ *
+ * @return map of property names to values for the specified service and configuration type
+ */
+ public Map<String, String> getConfigurationProperties(String service, String type) {
+ Map<String, String> configMap = new HashMap<>();
+ Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type);
+ if (configProperties != null) {
+ for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
+ configMap.put(configProperty.getKey(), configProperty.getValue().getValue());
+ }
+ }
+ return configMap;
+ }
+
+ public Map<String, ConfigProperty> getConfigurationPropertiesWithMetadata(String service, String type) {
+ return serviceConfigurations.get(service).get(type);
+ }
+
+ /**
+ * Get all required config properties for the specified service.
+ *
+ * @param service service name
+ *
+ * @return collection of all required properties for the given service
+ */
+ public Collection<ConfigProperty> getRequiredConfigurationProperties(String service) {
+ Collection<ConfigProperty> requiredConfigProperties = new HashSet<>();
+ Map<String, Map<String, ConfigProperty>> serviceProperties = requiredServiceConfigurations.get(service);
+ if (serviceProperties != null) {
+ for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : serviceProperties.entrySet()) {
+ requiredConfigProperties.addAll(typePropertiesEntry.getValue().values());
+ }
+ }
+ return requiredConfigProperties;
+ }
+
+ /**
+ * Get required config properties for the specified service which belong to the specified property type.
+ *
+ * @param service service name
+ * @param propertyType property type
+ *
+ * @return collection of required properties for the given service and property type
+ */
+ public Collection<ConfigProperty> getRequiredConfigurationProperties(String service, org.apache.ambari.server.state.PropertyInfo.PropertyType propertyType) {
+ Collection<ConfigProperty> matchingProperties = new HashSet<>();
+ Map<String, Map<String, ConfigProperty>> requiredProperties = requiredServiceConfigurations.get(service);
+ if (requiredProperties != null) {
+ for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : requiredProperties.entrySet()) {
+ for (ConfigProperty configProperty : typePropertiesEntry.getValue().values()) {
+ if (configProperty.getPropertyTypes().contains(propertyType)) {
+ matchingProperties.add(configProperty);
+ }
+ }
+
+ }
+ }
+ return matchingProperties;
+ }
+
+ public boolean isPasswordProperty(String service, String type, String propertyName) {
+ return (serviceConfigurations.containsKey(service) &&
+ serviceConfigurations.get(service).containsKey(type) &&
+ serviceConfigurations.get(service).get(type).containsKey(propertyName) &&
+ serviceConfigurations.get(service).get(type).get(propertyName).getPropertyTypes().
+ contains(org.apache.ambari.server.state.PropertyInfo.PropertyType.PASSWORD));
+ }
+
+ //todo
+ public Map<String, String> getStackConfigurationProperties(String type) {
+ Map<String, String> configMap = new HashMap<>();
+ Map<String, ConfigProperty> configProperties = stackConfigurations.get(type);
+ if (configProperties != null) {
+ for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
+ configMap.put(configProperty.getKey(), configProperty.getValue().getValue());
+ }
+ }
+ return configMap;
+ }
+
+ public boolean isKerberosPrincipalNameProperty(String service, String type, String propertyName) {
+ return (serviceConfigurations.containsKey(service) &&
+ serviceConfigurations.get(service).containsKey(type) &&
+ serviceConfigurations.get(service).get(type).containsKey(propertyName) &&
+ serviceConfigurations.get(service).get(type).get(propertyName).getPropertyTypes().
+ contains(org.apache.ambari.server.state.PropertyInfo.PropertyType.KERBEROS_PRINCIPAL));
+ }
+ /**
+ * Get config attributes for the specified service and configuration type.
+ *
+ * @param service service name
+ * @param type configuration type
+ *
+ * @return map of attribute names to map of property names to attribute values
+ * for the specified service and configuration type
+ */
+ public Map<String, Map<String, String>> getConfigurationAttributes(String service, String type) {
+ Map<String, Map<String, String>> attributesMap = new HashMap<>();
+ Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type);
+ if (configProperties != null) {
+ for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
+ String propertyName = configProperty.getKey();
+ Map<String, String> propertyAttributes = configProperty.getValue().getAttributes();
+ if (propertyAttributes != null) {
+ for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) {
+ String attributeName = propertyAttribute.getKey();
+ String attributeValue = propertyAttribute.getValue();
+ if (attributeValue != null) {
+ Map<String, String> attributes = attributesMap.get(attributeName);
+ if (attributes == null) {
+ attributes = new HashMap<>();
+ attributesMap.put(attributeName, attributes);
+ }
+ attributes.put(propertyName, attributeValue);
+ }
+ }
+ }
+ }
+ }
+ return attributesMap;
+ }
+
+ //todo:
+ public Map<String, Map<String, String>> getStackConfigurationAttributes(String type) {
+ Map<String, Map<String, String>> attributesMap = new HashMap<>();
+ Map<String, ConfigProperty> configProperties = stackConfigurations.get(type);
+ if (configProperties != null) {
+ for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) {
+ String propertyName = configProperty.getKey();
+ Map<String, String> propertyAttributes = configProperty.getValue().getAttributes();
+ if (propertyAttributes != null) {
+ for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) {
+ String attributeName = propertyAttribute.getKey();
+ String attributeValue = propertyAttribute.getValue();
+ Map<String, String> attributes = attributesMap.get(attributeName);
+ if (attributes == null) {
+ attributes = new HashMap<>();
+ attributesMap.put(attributeName, attributes);
+ }
+ attributes.put(propertyName, attributeValue);
+ }
+ }
+ }
+ }
+ return attributesMap;
+ }
+
+ /**
+ * Get the service for the specified component.
+ *
+ * @param component component name
+ *
+ * @return service name that contains tha specified component
+ */
+ public String getServiceForComponent(String component) {
+ return componentService.get(component);
+ }
+
+ /**
+ * Get the names of the services which contains the specified components.
+ *
+ * @param components collection of components
+ *
+ * @return collection of services which contain the specified components
+ */
+ public Collection<String> getServicesForComponents(Collection<String> components) {
+ Set<String> services = new HashSet<>();
+ for (String component : components) {
+ services.add(getServiceForComponent(component));
+ }
+
+ return services;
+ }
+
+ /**
+ * Obtain the service name which corresponds to the specified configuration.
+ *
+ * @param config configuration type
+ *
+ * @return name of service which corresponds to the specified configuration type
+ */
+ public String getServiceForConfigType(String config) {
+ for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) {
+ Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue();
+ String serviceName = entry.getKey();
+ if (typeMap.containsKey(config) && !getExcludedConfigurationTypes(serviceName).contains(config)) {
+ return serviceName;
+ }
+ }
+ throw new IllegalArgumentException(
+ "Specified configuration type is not associated with any service: " + config);
+ }
+
+ /**
+ * Return the dependencies specified for the given component.
+ *
+ * @param component component to get dependency information for
+ *
+ * @return collection of dependency information for the specified component
+ */
+ //todo: full dependency graph
+ public Collection<DependencyInfo> getDependenciesForComponent(String component) {
+ return dependencies.containsKey(component) ? dependencies.get(component) :
+ Collections.emptySet();
+ }
+
+ /**
+ * Get the service, if any, that a component dependency is conditional on.
+ *
+ * @param dependency dependency to get conditional service for
+ *
+ * @return conditional service for provided component or null if dependency
+ * is not conditional on a service
+ */
+ public String getConditionalServiceForDependency(DependencyInfo dependency) {
+ return dependencyConditionalServiceMap.get(dependency);
+ }
+
+ public String getExternalComponentConfig(String component) {
+ return dbDependencyInfo.get(component);
+ }
+
+ /**
+ * Obtain the required cardinality for the specified component.
+ */
+ public Cardinality getCardinality(String component) {
+ return new Cardinality(cardinalityRequirements.get(component));
+ }
+
+ /**
+ * Obtain auto-deploy information for the specified component.
+ */
+ public AutoDeployInfo getAutoDeployInfo(String component) {
+ return componentAutoDeployInfo.get(component);
+ }
+
+ public boolean isMasterComponent(String component) {
+ return masterComponents.contains(component);
+ }
+
+ public Configuration getConfiguration(Collection<String> services) {
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<>();
+ Map<String, Map<String, String>> properties = new HashMap<>();
+
+ for (String service : services) {
+ Collection<String> serviceConfigTypes = getConfigurationTypes(service);
+ for (String type : serviceConfigTypes) {
+ Map<String, String> typeProps = properties.get(type);
+ if (typeProps == null) {
+ typeProps = new HashMap<>();
+ properties.put(type, typeProps);
+ }
+ typeProps.putAll(getConfigurationProperties(service, type));
+
+ Map<String, Map<String, String>> stackTypeAttributes = getConfigurationAttributes(service, type);
+ if (!stackTypeAttributes.isEmpty()) {
+ if (! attributes.containsKey(type)) {
+ attributes.put(type, new HashMap<>());
+ }
+ Map<String, Map<String, String>> typeAttributes = attributes.get(type);
+ for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) {
+ String attributeName = attribute.getKey();
+ Map<String, String> attributeProps = typeAttributes.get(attributeName);
+ if (attributeProps == null) {
+ attributeProps = new HashMap<>();
+ typeAttributes.put(attributeName, attributeProps);
+ }
+ attributeProps.putAll(attribute.getValue());
+ }
+ }
+ }
+ }
+ return new Configuration(properties, attributes);
+ }
+
+ public Configuration getConfiguration() {
+ Map<String, Map<String, Map<String, String>>> stackAttributes = new HashMap<>();
+ Map<String, Map<String, String>> stackConfigs = new HashMap<>();
+
+ for (String service : getServices()) {
+ for (String type : getAllConfigurationTypes(service)) {
+ Map<String, String> typeProps = stackConfigs.get(type);
+ if (typeProps == null) {
+ typeProps = new HashMap<>();
+ stackConfigs.put(type, typeProps);
+ }
+ typeProps.putAll(getConfigurationProperties(service, type));
+
+ Map<String, Map<String, String>> stackTypeAttributes = getConfigurationAttributes(service, type);
+ if (!stackTypeAttributes.isEmpty()) {
+ if (! stackAttributes.containsKey(type)) {
+ stackAttributes.put(type, new HashMap<>());
+ }
+ Map<String, Map<String, String>> typeAttrs = stackAttributes.get(type);
+ for (Map.Entry<String, Map<String, String>> attribute : stackTypeAttributes.entrySet()) {
+ String attributeName = attribute.getKey();
+ Map<String, String> attributes = typeAttrs.get(attributeName);
+ if (attributes == null) {
+ attributes = new HashMap<>();
+ typeAttrs.put(attributeName, attributes);
+ }
+ attributes.putAll(attribute.getValue());
+ }
+ }
+ }
+ }
+ return new Configuration(stackConfigs, stackAttributes);
+ }
+
+ /**
+ * Parse components for the specified service from the stack definition.
+ *
+ * @param service service name
+ *
+ * @throws AmbariException an exception occurred getting components from the stack definition
+ */
+ private void parseComponents(String service) throws AmbariException{
+ Collection<String> componentSet = new HashSet<>();
+
+ Set<StackServiceComponentResponse> components = controller.getStackComponents(
+ Collections.singleton(new StackServiceComponentRequest(name, version, service, null)));
+
+ // stack service components
+ for (StackServiceComponentResponse component : components) {
+ String componentName = component.getComponentName();
+ componentSet.add(componentName);
+ componentService.put(componentName, service);
+ String cardinality = component.getCardinality();
+ if (cardinality != null) {
+ cardinalityRequirements.put(componentName, cardinality);
+ }
+ AutoDeployInfo autoDeploy = component.getAutoDeploy();
+ if (autoDeploy != null) {
+ componentAutoDeployInfo.put(componentName, autoDeploy);
+ }
+
+ // populate component dependencies
+ //todo: remove usage of AmbariMetaInfo
+ Collection<DependencyInfo> componentDependencies = controller.getAmbariMetaInfo().getComponentDependencies(
+ name, version, service, componentName);
+
+ if (componentDependencies != null && ! componentDependencies.isEmpty()) {
+ dependencies.put(componentName, componentDependencies);
+ }
+ if (component.isMaster()) {
+ masterComponents.add(componentName);
+ }
+ }
+ serviceComponents.put(service, componentSet);
+ }
+
+ /**
+ * Parse configurations for the specified service from the stack definition.
+ *
+ * @param stackService service to parse the stack configuration for
+ *
+ * @throws AmbariException an exception occurred getting configurations from the stack definition
+ */
+ private void parseConfigurations(StackServiceResponse stackService) throws AmbariException {
+ String service = stackService.getServiceName();
+ Map<String, Map<String, ConfigProperty>> mapServiceConfig = new HashMap<>();
+ Map<String, Map<String, ConfigProperty>> mapRequiredServiceConfig = new HashMap<>();
+
+
+ serviceConfigurations.put(service, mapServiceConfig);
+ requiredServiceConfigurations.put(service, mapRequiredServiceConfig);
+
+ Set<ReadOnlyConfigurationResponse> serviceConfigs = controller.getStackConfigurations(
+ Collections.singleton(new StackConfigurationRequest(name, version, service, null)));
+ Set<ReadOnlyConfigurationResponse> stackLevelConfigs = controller.getStackLevelConfigurations(
+ Collections.singleton(new StackLevelConfigurationRequest(name, version, null)));
+ serviceConfigs.addAll(stackLevelConfigs);
+
+ // shouldn't have any required properties in stack level configuration
+ for (ReadOnlyConfigurationResponse config : serviceConfigs) {
+ ConfigProperty configProperty = new ConfigProperty(config);
+ String type = configProperty.getType();
+
+ Map<String, ConfigProperty> mapTypeConfig = mapServiceConfig.get(type);
+ if (mapTypeConfig == null) {
+ mapTypeConfig = new HashMap<>();
+ mapServiceConfig.put(type, mapTypeConfig);
+ }
+
+ mapTypeConfig.put(config.getPropertyName(), configProperty);
+ if (config.isRequired()) {
+ Map<String, ConfigProperty> requiredTypeConfig = mapRequiredServiceConfig.get(type);
+ if (requiredTypeConfig == null) {
+ requiredTypeConfig = new HashMap<>();
+ mapRequiredServiceConfig.put(type, requiredTypeConfig);
+ }
+ requiredTypeConfig.put(config.getPropertyName(), configProperty);
+ }
+ }
+
+ // So far we added only config types that have properties defined
+ // in stack service definition. Since there might be config types
+ // with no properties defined we need to add those separately
+ Set<String> configTypes = stackService.getConfigTypes().keySet();
+ for (String configType: configTypes) {
+ if (!mapServiceConfig.containsKey(configType)) {
+ mapServiceConfig.put(configType, Collections.emptyMap());
+ }
+ }
+ }
+
+ private void parseStackConfigurations () throws AmbariException {
+
+ Set<ReadOnlyConfigurationResponse> stackLevelConfigs = controller.getStackLevelConfigurations(
+ Collections.singleton(new StackLevelConfigurationRequest(name, version, null)));
+
+ for (ReadOnlyConfigurationResponse config : stackLevelConfigs) {
+ ConfigProperty configProperty = new ConfigProperty(config);
+ String type = configProperty.getType();
+
+ Map<String, ConfigProperty> mapTypeConfig = stackConfigurations.get(type);
+ if (mapTypeConfig == null) {
+ mapTypeConfig = new HashMap<>();
+ stackConfigurations.put(type, mapTypeConfig);
+ }
+
+ mapTypeConfig.put(config.getPropertyName(),
+ configProperty);
+ }
+ }
+
+ /**
+ * Obtain the excluded configuration types from the StackServiceResponse
+ *
+ * @param stackServiceResponse the response object associated with this stack service
+ */
+ private void parseExcludedConfigurations(StackServiceResponse stackServiceResponse) {
+ excludedConfigurationTypes.put(stackServiceResponse.getServiceName(), stackServiceResponse.getExcludedConfigTypes());
+ }
+
+ /**
+ * Register conditional dependencies.
+ */
+ //todo: This information should be specified in the stack definition.
+ void registerConditionalDependencies() {
+ dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database");
+ }
+
+ /**
+ * Contains a configuration property's value and attributes.
+ */
+ public static class ConfigProperty {
+ private ValueAttributesInfo propertyValueAttributes = null;
+ private String name;
+ private String value;
+ private Map<String, String> attributes;
+ private Set<org.apache.ambari.server.state.PropertyInfo.PropertyType> propertyTypes;
+ private String type;
+ private Set<PropertyDependencyInfo> dependsOnProperties =
+ Collections.emptySet();
+
+ public ConfigProperty(ReadOnlyConfigurationResponse config) {
+ this.name = config.getPropertyName();
+ this.value = config.getPropertyValue();
+ this.attributes = config.getPropertyAttributes();
+ this.propertyTypes = config.getPropertyType();
+ this.type = normalizeType(config.getType());
+ this.dependsOnProperties = config.getDependsOnProperties();
+ this.propertyValueAttributes = config.getPropertyValueAttributes();
+ }
+
+ public ConfigProperty(String type, String name, String value) {
+ this.type = type;
+ this.name = name;
+ this.value = value;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public Set<org.apache.ambari.server.state.PropertyInfo.PropertyType> getPropertyTypes() {
+ return propertyTypes;
+ }
+
+ public void setPropertyTypes(Set<PropertyInfo.PropertyType> propertyTypes) {
+ this.propertyTypes = propertyTypes;
+ }
+
+ public Map<String, String> getAttributes() {
+ return attributes;
+ }
+
+ public void setAttributes(Map<String, String> attributes) {
+ this.attributes = attributes;
+ }
+
+ Set<PropertyDependencyInfo> getDependsOnProperties() {
+ return this.dependsOnProperties;
+ }
+
+ private String normalizeType(String type) {
+ //strip .xml from type
+ if (type.endsWith(".xml")) {
+ type = type.substring(0, type.length() - 4);
+ }
+ return type;
+ }
+
+ public ValueAttributesInfo getPropertyValueAttributes() {
+ return propertyValueAttributes;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
index 8b7cb67..37725b1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java
@@ -17,15 +17,15 @@
*/
package org.apache.ambari.server.controller.internal;
-import static org.apache.commons.lang.StringUtils.isBlank;
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.validators.UnitValidatedProperty;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
-import org.apache.ambari.server.topology.ClusterTopology;
-import org.apache.ambari.server.topology.validators.UnitValidatedProperty;
+import static org.apache.commons.lang.StringUtils.isBlank;
/**
* I append the stack defined unit to the original property value.
@@ -48,8 +48,9 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda
public String updateForClusterCreate(String propertyName,
String origValue,
Map<String, Map<String, String>> properties,
- ClusterTopology topology) {
- PropertyUnit stackUnit = PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType, propertyName);
+ ClusterTopology topology,
+ ConfigurationContext configurationContext) {
+ PropertyUnit stackUnit = PropertyUnit.of(configurationContext.getStack(), serviceName, configType, propertyName);
PropertyValue value = PropertyValue.of(propertyName, origValue);
if (value.hasUnit(stackUnit)) {
return value.toString();
@@ -61,7 +62,8 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda
}
@Override
- public Collection<String> getRequiredHostGroups(String propertyName, String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) {
+ public Collection<String> getRequiredHostGroups(String propertyName, String origValue, Map<String, Map<String, String>> properties,
+ ClusterTopology topology, ConfigurationContext configurationContext) {
return Collections.emptySet();
}
@@ -69,18 +71,18 @@ public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpda
private static final String DEFAULT_UNIT = "m";
private final String unit;
- public static PropertyUnit of(Stack stack, UnitValidatedProperty property) {
+ public static PropertyUnit of(StackV2 stack, UnitValidatedProperty property) {
return PropertyUnit.of(stack, property.getServiceName(), property.getConfigType(), property.getPropertyName());
}
- public static PropertyUnit of(Stack stack, String serviceName, String configType, String propertyName) {
+ public static PropertyUnit of(StackV2 stack, String serviceName, String configType, String propertyName) {
return new PropertyUnit(
stackUnit(stack, serviceName, configType, propertyName)
.map(PropertyUnit::toJvmUnit)
.orElse(DEFAULT_UNIT));
}
- private static Optional<String> stackUnit(Stack stack, String serviceName, String configType, String propertyName) {
+ private static Optional<String> stackUnit(StackV2 stack, String serviceName, String configType, String propertyName) {
try {
return Optional.ofNullable(
stack.getConfigurationPropertiesWithMetadata(serviceName, configType)
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 5af2a86..3a65662 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -18,82 +18,36 @@
package org.apache.ambari.server.topology;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.ClusterNotFoundException;
-import org.apache.ambari.server.DuplicateResourceException;
-import org.apache.ambari.server.Role;
-import org.apache.ambari.server.RoleCommand;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Striped;
+import com.google.inject.Provider;
+import org.apache.ambari.server.*;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleCommandFactory;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.AmbariServer;
-import org.apache.ambari.server.controller.ClusterRequest;
-import org.apache.ambari.server.controller.ConfigGroupRequest;
-import org.apache.ambari.server.controller.ConfigurationRequest;
-import org.apache.ambari.server.controller.RequestStatusResponse;
-import org.apache.ambari.server.controller.ServiceComponentHostRequest;
-import org.apache.ambari.server.controller.ServiceComponentRequest;
-import org.apache.ambari.server.controller.ServiceRequest;
-import org.apache.ambari.server.controller.internal.AbstractResourceProvider;
-import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
-import org.apache.ambari.server.controller.internal.ConfigGroupResourceProvider;
-import org.apache.ambari.server.controller.internal.HostComponentResourceProvider;
-import org.apache.ambari.server.controller.internal.HostResourceProvider;
-import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
-import org.apache.ambari.server.controller.internal.RequestImpl;
-import org.apache.ambari.server.controller.internal.ServiceResourceProvider;
-import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider;
+import org.apache.ambari.server.controller.*;
+import org.apache.ambari.server.controller.internal.*;
import org.apache.ambari.server.controller.predicate.EqualsPredicate;
import org.apache.ambari.server.controller.spi.ClusterController;
import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Request;
-import org.apache.ambari.server.controller.spi.RequestStatus;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.security.authorization.AuthorizationException;
-import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.ConfigFactory;
-import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.state.DesiredConfig;
-import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.RepositoryType;
-import org.apache.ambari.server.state.SecurityType;
-import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.*;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.RetryHelper;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Striped;
-import com.google.inject.Provider;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
/**
@@ -130,6 +84,7 @@ public class AmbariContext {
private static HostRoleCommandFactory hostRoleCommandFactory;
private static HostResourceProvider hostResourceProvider;
+ private static ServiceGroupResourceProvider serviceGroupResourceProvider;
private static ServiceResourceProvider serviceResourceProvider;
private static ComponentResourceProvider componentResourceProvider;
private static HostComponentResourceProvider hostComponentResourceProvider;
@@ -200,95 +155,12 @@ public class AmbariContext {
return getController().getActionManager().getTasks(ids);
}
- public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType,
- String repoVersionString, Long repoVersionId) {
- Stack stack = topology.getBlueprint().getStack();
- StackId stackId = new StackId(stack.getName(), stack.getVersion());
-
- RepositoryVersionEntity repoVersion = null;
- if (StringUtils.isEmpty(repoVersionString) && null == repoVersionId) {
- List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId);
-
- if (stackRepoVersions.isEmpty()) {
- // !!! no repos, try to get the version for the stack
- VersionDefinitionResourceProvider vdfProvider = getVersionDefinitionResourceProvider();
-
- Map<String, Object> properties = new HashMap<>();
- properties.put(VersionDefinitionResourceProvider.VERSION_DEF_AVAILABLE_DEFINITION, stackId.toString());
-
- Request request = new RequestImpl(Collections.<String>emptySet(),
- Collections.singleton(properties), Collections.<String, String>emptyMap(), null);
-
- Long defaultRepoVersionId = null;
-
- try {
- RequestStatus requestStatus = vdfProvider.createResources(request);
- if (!requestStatus.getAssociatedResources().isEmpty()) {
- Resource resource = requestStatus.getAssociatedResources().iterator().next();
- defaultRepoVersionId = (Long) resource.getPropertyValue(VersionDefinitionResourceProvider.VERSION_DEF_ID);
- }
- } catch (Exception e) {
- throw new IllegalArgumentException(String.format(
- "Failed to create a default repository version definition for stack %s. "
- + "This typically is a result of not loading the stack correctly or being able "
- + "to load information about released versions. Create a repository version "
- + " and try again.", stackId), e);
- }
-
- repoVersion = repositoryVersionDAO.findByPK(defaultRepoVersionId);
- // !!! better not!
- if (null == repoVersion) {
- throw new IllegalArgumentException(String.format(
- "Failed to load the default repository version definition for stack %s. "
- + "Check for a valid repository version and try again.", stackId));
- }
-
- } else if (stackRepoVersions.size() > 1) {
-
- Function<RepositoryVersionEntity, String> function = new Function<RepositoryVersionEntity, String>() {
- @Override
- public String apply(RepositoryVersionEntity input) {
- return input.getVersion();
- }
- };
-
- Collection<String> versions = Collections2.transform(stackRepoVersions, function);
-
- throw new IllegalArgumentException(String.format("Several repositories were found for %s: %s. Specify the version"
- + " with '%s'", stackId, StringUtils.join(versions, ", "), ProvisionClusterRequest.REPO_VERSION_PROPERTY));
- } else {
- repoVersion = stackRepoVersions.get(0);
- LOG.warn("Cluster is being provisioned using the single matching repository version {}", repoVersion.getVersion());
- }
- } else if (null != repoVersionId){
- repoVersion = repositoryVersionDAO.findByPK(repoVersionId);
-
- if (null == repoVersion) {
- throw new IllegalArgumentException(String.format(
- "Could not identify repository version with repository version id %s for installing services. "
- + "Specify a valid repository version id with '%s'",
- repoVersionId, ProvisionClusterRequest.REPO_VERSION_ID_PROPERTY));
- }
- } else {
- repoVersion = repositoryVersionDAO.findByStackAndVersion(stackId, repoVersionString);
-
- if (null == repoVersion) {
- throw new IllegalArgumentException(String.format(
- "Could not identify repository version with stack %s and version %s for installing services. "
- + "Specify a valid version with '%s'",
- stackId, repoVersionString, ProvisionClusterRequest.REPO_VERSION_PROPERTY));
- }
- }
+ public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType) {
- // only use a STANDARD repo when creating a new cluster
- if (repoVersion.getType() != RepositoryType.STANDARD) {
- throw new IllegalArgumentException(String.format(
- "Unable to create a cluster using the following repository since it is not a STANDARD type: %s",
- repoVersion));
- }
+ StackV2 stack = topology.getBlueprint().getStacks().iterator().next();
createAmbariClusterResource(clusterName, stack.getName(), stack.getVersion(), securityType);
- createAmbariServiceAndComponentResources(topology, clusterName, stackId, repoVersion.getId());
+ createAmbariServiceAndComponentResources(topology, clusterName);
}
public void createAmbariClusterResource(String clusterName, String stackName, String stackVersion, SecurityType securityType) {
@@ -314,34 +186,56 @@ public class AmbariContext {
}
}
- public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName,
- StackId stackId, Long repositoryVersionId) {
- Collection<String> services = topology.getBlueprint().getServices();
+ public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName) {
- try {
- Cluster cluster = getController().getClusters().getCluster(clusterName);
- services.removeAll(cluster.getServices().keySet());
- } catch (AmbariException e) {
- throw new RuntimeException("Failed to persist service and component resources: " + e, e);
- }
+ Collection<ServiceGroup> serviceGroups = topology.getBlueprint().getServiceGroups();
+ Set<ServiceGroupRequest> serviceGroupRequests = new HashSet<>();
Set<ServiceRequest> serviceRequests = new HashSet<>();
Set<ServiceComponentRequest> componentRequests = new HashSet<>();
- for (String service : services) {
- String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service);
- serviceRequests.add(new ServiceRequest(clusterName, null, service, service,
- repositoryVersionId, null, credentialStoreEnabled, null));
-
- for (String component : topology.getBlueprint().getComponents(service)) {
- String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component);
- componentRequests.add(new ServiceComponentRequest(clusterName, service, component, null, recoveryEnabled));
+
+ for (ServiceGroup serviceGroup : serviceGroups) {
+ serviceGroupRequests.add(new ServiceGroupRequest(clusterName, serviceGroup.getName()));
+
+ for (Service service : serviceGroup.getServices()) {
+ String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service.getType());
+
+ StackV2 stack = service.getStack();
+ StackId stackId = new StackId(stack.getName(), stack.getVersion());
+ RepositoryVersionEntity repoVersion = repositoryVersionDAO.findByStackAndVersion(stackId, stack.getRepoVersion());
+
+ if (null == repoVersion) {
+ throw new IllegalArgumentException(String.format(
+ "Could not identify repository version with stack %s and version %s for installing services. "
+ + "Specify a valid version with '%s'",
+ stackId, stack.getRepoVersion(), ProvisionClusterRequest.REPO_VERSION_PROPERTY));
+ }
+
+ // only use a STANDARD repo when creating a new cluster
+ if (repoVersion.getType() != RepositoryType.STANDARD) {
+ throw new IllegalArgumentException(String.format(
+ "Unable to create a cluster using the following repository since it is not a STANDARD type: %s",
+ repoVersion));
+ }
+
+ serviceRequests.add(new ServiceRequest(clusterName, serviceGroup.getName(), service.getType(), service.getName(),
+ repoVersion.getId(), null, credentialStoreEnabled, null));
+
+ for (ComponentV2 component : topology.getBlueprint().getComponents(service)) {
+ String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(component);
+ componentRequests.add(new ServiceComponentRequest(clusterName, serviceGroup.getName(), service.getName(),
+ component.getName(), null, recoveryEnabled));
+ }
}
+
}
try {
+ getServiceGroupResourceProvider().createServiceGroups(serviceGroupRequests);
getServiceResourceProvider().createServices(serviceRequests);
getComponentResourceProvider().createComponents(componentRequests);
} catch (AmbariException | AuthorizationException e) {
throw new RuntimeException("Failed to persist service and component resources: " + e, e);
}
+
// set all services state to INSTALLED->STARTED
// this is required so the user can start failed services at the service level
Map<String, Object> installProps = new HashMap<>();
@@ -351,20 +245,20 @@ public class AmbariContext {
startProps.put(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID, "STARTED");
startProps.put(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName);
Predicate predicate = new EqualsPredicate<>(
- ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName);
+ ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName);
try {
getServiceResourceProvider().updateResources(
- new RequestImpl(null, Collections.singleton(installProps), null, null), predicate);
+ new RequestImpl(null, Collections.singleton(installProps), null, null), predicate);
getServiceResourceProvider().updateResources(
- new RequestImpl(null, Collections.singleton(startProps), null, null), predicate);
+ new RequestImpl(null, Collections.singleton(startProps), null, null), predicate);
} catch (Exception e) {
// just log as this won't prevent cluster from being provisioned correctly
LOG.error("Unable to update state of services during cluster provision: " + e, e);
}
}
- public void createAmbariHostResources(long clusterId, String hostName, Map<String, Collection<String>> components) {
+ public void createAmbariHostResources(long clusterId, String hostName, Map<Service, Collection<ComponentV2>> components) {
Host host;
try {
host = getController().getClusters().getHost(hostName);
@@ -398,13 +292,14 @@ public class AmbariContext {
final Set<ServiceComponentHostRequest> requests = new HashSet<>();
- for (Map.Entry<String, Collection<String>> entry : components.entrySet()) {
- String service = entry.getKey();
- for (String component : entry.getValue()) {
+ for (Map.Entry<Service, Collection<ComponentV2>> entry : components.entrySet()) {
+ Service service = entry.getKey();
+ for (ComponentV2 component : entry.getValue()) {
//todo: handle this in a generic manner. These checks are all over the code
try {
- if (cluster.getService(service) != null && !component.equals("AMBARI_SERVER")) {
- requests.add(new ServiceComponentHostRequest(clusterName, null, service, component, hostName, null));
+ if (cluster.getService(service.getName()) != null && !component.getType().equals("AMBARI_SERVER")) {
+ requests.add(new ServiceComponentHostRequest(clusterName, service.getServiceGroup().getName(),
+ service.getName(), component.getName(), hostName, null));
}
} catch(AmbariException se) {
LOG.warn("Service already deleted from cluster: {}", service);
@@ -718,34 +613,32 @@ public class AmbariContext {
* and the hosts associated with the host group are assigned to the config group.
*/
private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String groupName) throws AmbariException {
- Map<String, Map<String, Config>> groupConfigs = new HashMap<>();
- Stack stack = topology.getBlueprint().getStack();
-
- // get the host-group config with cluster creation template overrides
- Configuration topologyHostGroupConfig = topology.
- getHostGroupInfo().get(groupName).getConfiguration();
-
- // only get user provided configuration for host group which includes only CCT/HG and BP/HG properties
- Map<String, Map<String, String>> userProvidedGroupProperties =
- topologyHostGroupConfig.getFullProperties(1);
-
- // iterate over topo host group configs which were defined in
- for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) {
- String type = entry.getKey();
- String service = stack.getServiceForConfigType(type);
- Config config = configFactory.createReadOnly(type, groupName, entry.getValue(), null);
- //todo: attributes
- Map<String, Config> serviceConfigs = groupConfigs.get(service);
- if (serviceConfigs == null) {
- serviceConfigs = new HashMap<>();
- groupConfigs.put(service, serviceConfigs);
+
+ Map<Service, Map<String, Config>> groupConfigs = new HashMap<>();
+
+
+ // only get user provided configuration for host group per service which includes only CCT/HG and BP/HG properties
+ Collection<Service> serviceConfigurations = topology.getHostGroupInfo().get(groupName).getServiceConfigs();
+ serviceConfigurations.forEach(service -> {
+ Map<String, Map<String, String>> userProvidedGroupProperties = service.getConfiguration().getProperties();
+
+ // iterate over topo host group configs which were defined in
+ for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) {
+ String type = entry.getKey();
+ Config config = configFactory.createReadOnly(type, groupName, entry.getValue(), null);
+ //todo: attributes
+ Map<String, Config> serviceConfigs = groupConfigs.get(service);
+ if (serviceConfigs == null) {
+ serviceConfigs = new HashMap<>();
+ groupConfigs.put(service, serviceConfigs);
+ }
+ serviceConfigs.put(type, config);
}
- serviceConfigs.put(type, config);
- }
+ });
String bpName = topology.getBlueprint().getName();
- for (Map.Entry<String, Map<String, Config>> entry : groupConfigs.entrySet()) {
- String service = entry.getKey();
+ for (Map.Entry<Service, Map<String, Config>> entry : groupConfigs.entrySet()) {
+ Service service = entry.getKey();
Map<String, Config> serviceConfigs = entry.getValue();
String absoluteGroupName = getConfigurationGroupName(bpName, groupName);
Collection<String> groupHosts;
@@ -771,7 +664,7 @@ public class AmbariContext {
});
ConfigGroupRequest request = new ConfigGroupRequest(null, clusterName,
- absoluteGroupName, service, service, "Host Group Configuration",
+ absoluteGroupName, service.getName(), service.getName(), "Host Group Configuration",
Sets.newHashSet(filteredGroupHosts), serviceConfigs);
// get the config group provider and create config group resource
@@ -826,6 +719,14 @@ public class AmbariContext {
return hostComponentResourceProvider;
}
+ private synchronized ServiceGroupResourceProvider getServiceGroupResourceProvider() {
+ if (serviceGroupResourceProvider == null) {
+ serviceGroupResourceProvider = (ServiceGroupResourceProvider) ClusterControllerHelper.
+ getClusterController().ensureResourceProvider(Resource.Type.ServiceGroup);
+ }
+ return serviceGroupResourceProvider;
+ }
+
private synchronized ServiceResourceProvider getServiceResourceProvider() {
if (serviceResourceProvider == null) {
serviceResourceProvider = (ServiceResourceProvider) ClusterControllerHelper.
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
index 404068d..c8860f6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
@@ -19,13 +19,7 @@
package org.apache.ambari.server.topology;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.inject.Inject;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ObjectNotFoundException;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -37,7 +31,7 @@ import org.apache.ambari.server.orm.dao.BlueprintDAO;
import org.apache.ambari.server.orm.entities.BlueprintEntity;
import org.apache.ambari.server.stack.NoSuchStackException;
-import com.google.inject.Inject;
+import java.util.*;
/**
* Create a Blueprint instance.
@@ -82,10 +76,10 @@ public class BlueprintFactory {
this.stackFactory = stackFactory;
}
- public Blueprint getBlueprint(String blueprintName) throws NoSuchStackException {
+ public BlueprintV2 getBlueprint(String blueprintName) throws NoSuchStackException {
BlueprintEntity entity = blueprintDAO.findByName(blueprintName);
//todo: just return null?
- return entity == null ? null : new BlueprintImpl(entity);
+ return entity == null ? null : new BlueprintImplV2(entity);
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java
index 9bde795..79456e5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImplV2.java
@@ -19,18 +19,20 @@
package org.apache.ambari.server.topology;
+import org.apache.ambari.server.controller.internal.StackV2;
+import org.apache.ambari.server.orm.entities.BlueprintEntity;
+
import java.util.Collection;
import java.util.List;
import java.util.Map;
-
-import org.apache.ambari.server.controller.StackV2;
-import org.apache.ambari.server.orm.entities.BlueprintEntity;
+import java.util.stream.Collectors;
/**
* Blueprint implementation.
*/
public class BlueprintImplV2 implements BlueprintV2 {
-
+ public BlueprintImplV2(BlueprintEntity e) {
+ }
@Override
public String getName() {
@@ -63,11 +65,38 @@ public class BlueprintImplV2 implements BlueprintV2 {
}
@Override
+ public Collection<String> getAllServiceTypes() {
+ return null;
+ }
+
+ @Override
+ public Collection<Service> getServicesByType(String serviceType) {
+ return getAllServices().stream().filter(
+ service -> service.getType().equalsIgnoreCase(serviceType)).collect(Collectors.toList());
+ }
+
+ @Override
+ public Collection<Service> getServicesFromServiceGroup(ServiceGroup serviceGroup, String serviceType) {
+ if (serviceType == null) {
+ return serviceGroup.getServices();
+ } else {
+ return serviceGroup.getServices().stream().filter(
+ service -> service.getType().equalsIgnoreCase(serviceType)).collect(Collectors.toList());
+ }
+ }
+
+ @Override
public Collection<ComponentV2> getComponents(Service service) {
return null;
}
@Override
+ public Collection<ComponentV2> getComponentsByType(Service service, String componentType) {
+ return getComponents(service).stream().filter(
+ compnoent -> compnoent.getType().equalsIgnoreCase(componentType)).collect(Collectors.toList());
+ }
+
+ @Override
public Collection<HostGroupV2> getHostGroupsForService(Service service) {
return null;
}
@@ -78,6 +107,7 @@ public class BlueprintImplV2 implements BlueprintV2 {
}
@Override
+ @Deprecated
public Configuration getConfiguration() {
return null;
}
@@ -88,7 +118,7 @@ public class BlueprintImplV2 implements BlueprintV2 {
}
@Override
- public String getRecoveryEnabled(String serviceName, String componentName) {
+ public String getRecoveryEnabled(ComponentV2 component) {
return null;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java
index 1fea966..3c71e41 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintV2.java
@@ -18,12 +18,13 @@
package org.apache.ambari.server.topology;
+import org.apache.ambari.server.controller.internal.StackV2;
+import org.apache.ambari.server.orm.entities.BlueprintEntity;
+
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import org.apache.ambari.server.controller.StackV2;
-import org.apache.ambari.server.orm.entities.BlueprintEntity;
/**
* Blueprint representation.
@@ -69,6 +70,29 @@ public interface BlueprintV2 {
*/
Collection<Service> getAllServices();
+
+ /**
+ * Get all of the service types represented in the blueprint.
+ *
+ * @return collection of all represented service types
+ */
+ Collection<String> getAllServiceTypes();
+
+ /**
+ * Get all of the services represented in the blueprint with a given type.
+ *
+ * @return collection of all represented services represented in the blueprint with a given type.
+ */
+ Collection<Service> getServicesByType(String serviceType);
+
+ /**
+ * Get services by type from a service group.
+ * @param serviceGroup
+ * @param serviceType
+ * @return
+ */
+ Collection<Service> getServicesFromServiceGroup(ServiceGroup serviceGroup, String serviceType);
+
/**
* Get the components that are included in the blueprint for the specified service.
*
@@ -78,6 +102,14 @@ public interface BlueprintV2 {
*/
Collection<ComponentV2> getComponents(Service service);
+ /**
+ * Get components by type from a service.
+ * @param service
+ * @param componentType
+ * @return
+ */
+ Collection<ComponentV2> getComponentsByType(Service service, String componentType);
+
/**
* Get the host groups which contain components for the specified service.
@@ -106,6 +138,7 @@ public interface BlueprintV2 {
*
* @return blueprint cluster scoped configuration
*/
+ @Deprecated
Configuration getConfiguration();
/**
@@ -121,12 +154,11 @@ public interface BlueprintV2 {
/**
* Get whether a component is enabled for auto start.
*
- * @param serviceName - Service name.
- * @param componentName - Component name.
+ * @param component - Component.
*
* @return null if value is not specified; true or false if specified.
*/
- String getRecoveryEnabled(String serviceName, String componentName);
+ String getRecoveryEnabled(ComponentV2 component);
/**
* Get whether a service is enabled for credential store use.
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 1a43b85..0244d7b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -18,14 +18,6 @@
package org.apache.ambari.server.topology;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
import org.apache.ambari.server.state.DependencyConditionInfo;
@@ -35,6 +27,11 @@ import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
/**
* Default blueprint validator.
*/
@@ -85,6 +82,8 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
@Override
public void validateRequiredProperties() throws InvalidTopologyException {
+ //TODO
+ // ConfigurationContext configurationContext = new ConfigurationContext(blueprint.getStacks().iterator().next(), blueprint.getConfiguration());
// we don't want to include default stack properties so we can't just use hostGroup full properties
Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
@@ -131,31 +130,31 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
" using existing db!");
}
}
- if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) {
- Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env");
- if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) {
- ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<>(blueprint.getHostGroupsForComponent(component));
- Set<String> givenHostGroups = new HashSet<>();
- givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active"));
- givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby"));
- if(givenHostGroups.size() != hostGroupsForComponent.size()) {
- throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent);
- }
- if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){
- for (HostGroup hostGroupForComponent : hostGroupsForComponent) {
- Iterator<String> itr = givenHostGroups.iterator();
- while(itr.hasNext()){
- if(itr.next().contains(hostGroupForComponent.getName())){
- itr.remove();
- }
- }
- }
- if(!givenHostGroups.isEmpty()){
- throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent);
- }
- }
- }
- }
+// if (configurationContext.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) {
+// Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env");
+// if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) {
+// ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<>(blueprint.getHostGroupsForComponent(component));
+// Set<String> givenHostGroups = new HashSet<>();
+// givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active"));
+// givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby"));
+// if(givenHostGroups.size() != hostGroupsForComponent.size()) {
+// throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent);
+// }
+// if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){
+// for (HostGroup hostGroupForComponent : hostGroupsForComponent) {
+// Iterator<String> itr = givenHostGroups.iterator();
+// while(itr.hasNext()){
+// if(itr.next().contains(hostGroupForComponent.getName())){
+// itr.remove();
+// }
+// }
+// }
+// if(!givenHostGroups.isEmpty()){
+// throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent);
+// }
+// }
+// }
+// }
if (component.equals("HIVE_METASTORE")) {
Map<String, String> hiveEnvConfig = clusterConfigurations.get("hive-env");
@@ -293,12 +292,13 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
Map<String, Map<String, String>> configProperties = blueprint.getConfiguration().getProperties();
Collection<String> cardinalityFailures = new HashSet<>();
//todo: don't hard code this HA logic here
- if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) &&
- (component.equals("SECONDARY_NAMENODE"))) {
- // override the cardinality for this component in an HA deployment,
- // since the SECONDARY_NAMENODE should not be started in this scenario
- cardinality = new Cardinality("0");
- }
+//TODO
+// if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) &&
+// (component.equals("SECONDARY_NAMENODE"))) {
+// // override the cardinality for this component in an HA deployment,
+// // since the SECONDARY_NAMENODE should not be started in this scenario
+// cardinality = new Cardinality("0");
+// }
int actualCount = blueprint.getHostGroupsForComponent(component).size();
if (! cardinality.isValidCount(actualCount)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 7bd377f..48f8fec 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -18,25 +18,11 @@
package org.apache.ambari.server.topology;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor;
import org.apache.ambari.server.controller.ClusterRequest;
import org.apache.ambari.server.controller.ConfigurationRequest;
-import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
-import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
-import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
-import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.internal.*;
import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.SecurityType;
@@ -46,6 +32,10 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
/**
* Responsible for cluster configuration.
*/
@@ -63,7 +53,6 @@ public class ClusterConfigurationRequest {
private ClusterTopology clusterTopology;
private BlueprintConfigurationProcessor configurationProcessor;
private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor;
- private Stack stack;
private boolean configureSecurity = false;
public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean configureSecurity) {
@@ -75,10 +64,11 @@ public class ClusterConfigurationRequest {
StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor) {
this.ambariContext = ambariContext;
this.clusterTopology = clusterTopology;
- Blueprint blueprint = clusterTopology.getBlueprint();
- this.stack = blueprint.getStack();
+ BlueprintV2 blueprint = clusterTopology.getBlueprint();
// set initial configuration (not topology resolved)
- this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology);
+ //TODO set up proper ConfigurationContext
+ ConfigurationContext configurationContext = new ConfigurationContext(blueprint.getStacks().iterator().next(), blueprint.getConfiguration());
+ this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology, configurationContext);
this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor;
removeOrphanConfigTypes();
if (setInitial) {
@@ -90,7 +80,7 @@ public class ClusterConfigurationRequest {
* Remove config-types from the given configuration if there is no any services related to them (except cluster-env and global).
*/
private void removeOrphanConfigTypes(Configuration configuration) {
- Blueprint blueprint = clusterTopology.getBlueprint();
+ BlueprintV2 blueprint = clusterTopology.getBlueprint();
Collection<String> configTypes = configuration.getAllConfigTypes();
for (String configType : configTypes) {
@@ -162,10 +152,11 @@ public class ClusterConfigurationRequest {
Set<String> updatedConfigTypes = new HashSet<>();
Cluster cluster = getCluster();
- Blueprint blueprint = clusterTopology.getBlueprint();
+ BlueprintV2 blueprint = clusterTopology.getBlueprint();
- Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
- Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
+ //Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getAllServices());
+ //Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
+ Map<String, Map<String, String>> stackDefaultProps = new Configuration(new HashMap<>(), new HashMap<>()).getProperties();
// add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of
// StageUtils.getClusterInfo()
@@ -176,7 +167,7 @@ public class ClusterConfigurationRequest {
// generate principals & keytabs for headless identities
AmbariContext.getController().getKerberosHelper()
.ensureHeadlessIdentities(cluster, existingConfigurations,
- new HashSet<>(blueprint.getServices()));
+ new HashSet(blueprint.getAllServices()));
// apply Kerberos specific configurations
Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
@@ -231,17 +222,17 @@ public class ClusterConfigurationRequest {
* @param blueprint the blueprint
* @return a map of service names to component names
*/
- private Map<String, Set<String>> createServiceComponentMap(Blueprint blueprint) {
+ private Map<String, Set<String>> createServiceComponentMap(BlueprintV2 blueprint) {
Map<String, Set<String>> serviceComponents = new HashMap<>();
- Collection<String> services = blueprint.getServices();
+ Collection<Service> services = blueprint.getAllServices();
if(services != null) {
- for (String service : services) {
- Collection<String> components = blueprint.getComponents(service);
- serviceComponents.put(service,
+ for (Service service : services) {
+ Collection<ComponentV2> components = blueprint.getComponents(service);
+ serviceComponents.put(service.getType(),
(components == null)
? Collections.emptySet()
- : new HashSet<>(blueprint.getComponents(service)));
+ : new HashSet(blueprint.getComponents(service)));
}
}
@@ -278,16 +269,16 @@ public class ClusterConfigurationRequest {
return propertyHasCustomValue;
}
- private Map<String, String> createComponentHostMap(Blueprint blueprint) {
+ private Map<String, String> createComponentHostMap(BlueprintV2 blueprint) {
Map<String, String> componentHostsMap = new HashMap<>();
- for (String service : blueprint.getServices()) {
- Collection<String> components = blueprint.getComponents(service);
- for (String component : components) {
- Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component);
+ for (Service service : blueprint.getAllServices()) {
+ Collection<ComponentV2> components = blueprint.getComponents(service);
+ for (ComponentV2 component : components) {
+ Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component.getType());
// retrieve corresponding clusterInfoKey for component using StageUtils
- String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component);
+ String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component.getType());
if (clusterInfoKey == null) {
- clusterInfoKey = component.toLowerCase() + "_hosts";
+ clusterInfoKey = component.getType().toLowerCase() + "_hosts";
}
componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ","));
}
@@ -300,7 +291,7 @@ public class ClusterConfigurationRequest {
try {
Cluster cluster = getCluster();
- Blueprint blueprint = clusterTopology.getBlueprint();
+ BlueprintV2 blueprint = clusterTopology.getBlueprint();
Configuration clusterConfiguration = clusterTopology.getConfiguration();
Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties();
@@ -352,16 +343,16 @@ public class ClusterConfigurationRequest {
//todo: also handle setting of host group scoped configuration which is updated by config processor
List<BlueprintServiceConfigRequest> configurationRequests = new LinkedList<>();
- Blueprint blueprint = clusterTopology.getBlueprint();
+ BlueprintV2 blueprint = clusterTopology.getBlueprint();
Configuration clusterConfiguration = clusterTopology.getConfiguration();
- for (String service : blueprint.getServices()) {
+ for (Service service : blueprint.getAllServices()) {
//todo: remove intermediate request type
// one bp config request per service
- BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service);
-
- for (String serviceConfigType : stack.getAllConfigurationTypes(service)) {
- Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(service);
+ BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service.getType());
+ StackV2 serviceStack = service.getStack();
+ for (String serviceConfigType : serviceStack.getAllConfigurationTypes(service.getType())) {
+ Set<String> excludedConfigTypes = serviceStack.getExcludedConfigurationTypes(service.getType());
if (!excludedConfigTypes.contains(serviceConfigType)) {
// skip handling of cluster-env here
if (! serviceConfigType.equals("cluster-env")) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/65d44cd5/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
index 69ccb61..f0d6e59 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
@@ -18,12 +18,13 @@
package org.apache.ambari.server.topology;
-import java.util.Collection;
-import java.util.Map;
-
import org.apache.ambari.server.controller.RequestStatusResponse;
+import org.apache.ambari.server.controller.internal.ConfigurationContext;
import org.apache.ambari.server.controller.internal.ProvisionAction;
+import java.util.Collection;
+import java.util.Map;
+
/**
* Represents a full cluster topology including all instance information as well as the associated
* blueprint which provides all abstract topology information.
@@ -49,7 +50,7 @@ public interface ClusterTopology {
*
* @return assocaited blueprint
*/
- Blueprint getBlueprint();
+ BlueprintV2 getBlueprint();
/**
* Get the cluster scoped configuration for the cluster.
@@ -58,8 +59,12 @@ public interface ClusterTopology {
*
* @return cluster scoped configuration
*/
+ @Deprecated
Configuration getConfiguration();
+
+ Collection<Service> getServiceConfigs();
+
/**
* Get host group information.
*
@@ -118,18 +123,18 @@ public interface ClusterTopology {
void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException;
/**
- * Determine if NameNode HA is enabled.
+ * Determine if NameNode HA is enabled within ConfigurationContext.
*
* @return true if NameNode HA is enabled; false otherwise
*/
- boolean isNameNodeHAEnabled();
+ boolean isNameNodeHAEnabled(ConfigurationContext configuration);
/**
- * Determine if Yarn ResourceManager HA is enabled.
+ * Determine if Yarn ResourceManager HA is enabled within ConfigurationContext.
*
* @return true if Yarn ResourceManager HA is enabled; false otherwise
*/
- boolean isYarnResourceManagerHAEnabled();
+ boolean isYarnResourceManagerHAEnabled(ConfigurationContext configuration);
/**
* Determine if the cluster is kerberos enabled.