You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/06/14 21:19:37 UTC
[06/26] ambari git commit: AMBARI-21180. Component command changes
must include version numbers for all services (ncole)
AMBARI-21180. Component command changes must include version numbers for all services (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5a1bd715
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5a1bd715
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5a1bd715
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 5a1bd715e11b2e3a79b139118a90b66eca32f025
Parents: 12202a2
Author: Nate Cole <nc...@hortonworks.com>
Authored: Mon Jun 5 17:39:42 2017 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Jun 6 10:33:14 2017 -0400
----------------------------------------------------------------------
.../actionmanager/ExecutionCommandWrapper.java | 65 +++++++++++---------
.../ambari/server/agent/ExecutionCommand.java | 57 ++++++++++++-----
.../AmbariCustomCommandExecutionHelper.java | 4 +-
.../AmbariManagementControllerImpl.java | 2 -
.../STORM/0.9.1/package/scripts/params_linux.py | 2 -
.../1.0.1.3.0/package/scripts/params_linux.py | 2 -
.../AmbariCustomCommandExecutionHelperTest.java | 11 ++--
7 files changed, 84 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 2ec09d9..efd609a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -35,6 +35,7 @@ import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -143,7 +144,8 @@ public class ExecutionCommandWrapper {
// now that the tags have been updated (if necessary), fetch the
// configurations
Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags();
- if (null != configurationTags && !configurationTags.isEmpty()) {
+
+ if (MapUtils.isNotEmpty(configurationTags)) {
Map<String, Map<String, String>> configProperties = configHelper
.getEffectiveConfigProperties(cluster, configurationTags);
@@ -182,43 +184,46 @@ public class ExecutionCommandWrapper {
executionCommand.getConfigurationAttributes().get(type));
}
}
+ }
- // set the repository version for the component this command is for -
- // always use the current desired version
- try {
- RepositoryVersionEntity repositoryVersion = null;
- String serviceName = executionCommand.getServiceName();
- if (!StringUtils.isEmpty(serviceName)) {
- Service service = cluster.getService(serviceName);
- if (null != service) {
- repositoryVersion = service.getDesiredRepositoryVersion();
- }
+ // set the repository version for the component this command is for -
+ // always use the current desired version
+ try {
+ RepositoryVersionEntity repositoryVersion = null;
+ String serviceName = executionCommand.getServiceName();
+ if (!StringUtils.isEmpty(serviceName)) {
+ Service service = cluster.getService(serviceName);
+ if (null != service) {
+ repositoryVersion = service.getDesiredRepositoryVersion();
+ }
- String componentName = executionCommand.getComponentName();
- if (!StringUtils.isEmpty(componentName)) {
- ServiceComponent serviceComponent = service.getServiceComponent(
- executionCommand.getComponentName());
+ String componentName = executionCommand.getComponentName();
+ if (!StringUtils.isEmpty(componentName)) {
+ ServiceComponent serviceComponent = service.getServiceComponent(
+ executionCommand.getComponentName());
- if (null != serviceComponent) {
- repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
- }
+ if (null != serviceComponent) {
+ repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
}
}
+ }
- if (null != repositoryVersion) {
- executionCommand.getCommandParams().put(KeyNames.VERSION,
- repositoryVersion.getVersion());
- executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION,
- repositoryVersion.getVersion());
- }
- } catch (ServiceNotFoundException serviceNotFoundException) {
- // it's possible that there are commands specified for a service where
- // the service doesn't exist yet
- LOG.warn(
- "The service {} is not installed in the cluster. No repository version will be sent for this command.",
- executionCommand.getServiceName());
+ if (null != repositoryVersion) {
+ executionCommand.getCommandParams().put(KeyNames.VERSION, repositoryVersion.getVersion());
+ executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION, repositoryVersion.getVersion());
}
+ } catch (ServiceNotFoundException serviceNotFoundException) {
+ // it's possible that there are commands specified for a service where
+ // the service doesn't exist yet
+ LOG.warn(
+ "The service {} is not installed in the cluster. No repository version will be sent for this command.",
+ executionCommand.getServiceName());
}
+
+ // set the desired versions of versionable components. This is safe even during an upgrade because
+ // we are "loading-late": components that have not yet upgraded in an EU will have the correct versions.
+ executionCommand.setComponentVersions(cluster);
+
} catch (ClusterNotFoundException cnfe) {
// it's possible that there are commands without clusters; in such cases,
// just return the de-serialized command and don't try to read configs
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 17b89b7..bd4c9dd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -26,8 +26,11 @@ import java.util.Set;
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.RoleCommand;
-import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -110,9 +113,6 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("localComponents")
private Set<String> localComponents = new HashSet<>();
- @SerializedName("availableServices")
- private Map<String, String> availableServices = new HashMap<>();
-
/**
* "true" or "false" indicating whether this
* service is enabled for credential store use.
@@ -153,6 +153,9 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("repositoryFile")
private CommandRepository commandRepository;
+ @SerializedName("componentVersionMap")
+ private Map<String, Map<String, String>> componentVersionMap = new HashMap<>();
+
public void setConfigurationCredentials(Map<String, Map<String, String>> configurationCredentials) {
this.configurationCredentials = configurationCredentials;
}
@@ -300,18 +303,6 @@ public class ExecutionCommand extends AgentCommand {
this.localComponents = localComponents;
}
- public Map<String, String> getAvailableServices() {
- return availableServices;
- }
-
- public void setAvailableServicesFromServiceInfoMap(Map<String, ServiceInfo> serviceInfoMap) {
- Map<String, String> serviceVersionMap = new HashMap<>();
- for (Map.Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) {
- serviceVersionMap.put(entry.getKey(), entry.getValue().getVersion());
- }
- availableServices = serviceVersionMap;
- }
-
public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() {
return configurationAttributes;
}
@@ -523,4 +514,38 @@ public class ExecutionCommand extends AgentCommand {
comment = "This should be replaced by a map of all service component versions")
String CURRENT_VERSION = "current_version";
}
+
+ /**
+ * @return
+ */
+ public Map<String, Map<String, String>> getComponentVersionMap() {
+ return componentVersionMap;
+ }
+
+ /**
+ * Used to set a map of {service -> { component -> version}}. This is necessary when performing
+ * an upgrade to correct build paths of required binaries.
+ * @param cluster the cluster from which to build the map
+ */
+ public void setComponentVersions(Cluster cluster) throws AmbariException {
+ Map<String, Map<String, String>> componentVersionMap = new HashMap<>();
+
+ for (Service service : cluster.getServices().values()) {
+ Map<String, String> componentMap = new HashMap<>();
+
+ boolean shouldSet = false;
+ for (ServiceComponent component : service.getServiceComponents().values()) {
+ if (component.isVersionAdvertised()) {
+ shouldSet = true;
+ componentMap.put(component.getName(), component.getDesiredVersion());
+ }
+ }
+
+ if (shouldSet) {
+ componentVersionMap.put(service.getName(), componentMap);
+ }
+ }
+
+ this.componentVersionMap = componentVersionMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index cabb9d6..1f3b4b0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -384,12 +384,12 @@ public class AmbariCustomCommandExecutionHelper {
execCmd.setCommandType(AgentCommandType.BACKGROUND_EXECUTION_COMMAND);
}
+ execCmd.setComponentVersions(cluster);
+
execCmd.setConfigurations(configurations);
execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
- execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()));
-
// Get the value of credential store enabled from the DB
Service clusterService = cluster.getService(serviceName);
execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 186a19e..329998b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2555,11 +2555,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
cluster.addSuspendedUpgradeParameters(commandParams, roleParams);
}
-
execCmd.setRoleParams(roleParams);
execCmd.setCommandParams(commandParams);
- execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()));
execCmd.setRepositoryFile(customCommandExecutionHelper.getCommandRepository(cluster, component, host));
hostParams.put(KeyNames.CURRENT_VERSION, repoVersion.getVersion());
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index 78ec165..664c582 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -394,8 +394,6 @@ if enable_ranger_storm:
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
-availableServices = config['availableServices']
-
hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
index 78ec165..664c582 100644
--- a/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py
@@ -394,8 +394,6 @@ if enable_ranger_storm:
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
-availableServices = config['availableServices']
-
hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
index a6c0c32..240517f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
@@ -70,7 +70,6 @@ import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
-import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.State;
@@ -79,6 +78,7 @@ import org.apache.ambari.server.state.ValueAttributesInfo;
import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
import org.apache.ambari.server.topology.TopologyManager;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.collections.MapUtils;
import org.easymock.Capture;
import org.easymock.EasyMock;
import org.easymock.EasyMockRule;
@@ -553,14 +553,15 @@ public class AmbariCustomCommandExecutionHelperTest {
ambariManagementController.createAction(actionRequest, requestProperties);
StackId stackId = clusters.getCluster("c1").getDesiredStackVersion();
- Map<String, ServiceInfo> services = ambariManagementController.getAmbariMetaInfo().getServices(stackId.getStackName(), stackId.getStackVersion());
Request request = requestCapture.getValue();
Stage stage = request.getStages().iterator().next();
List<ExecutionCommandWrapper> commands = stage.getExecutionCommands("c1-c6401");
ExecutionCommand command = commands.get(0).getExecutionCommand();
- for (String service : services.keySet()) {
- Assert.assertEquals(command.getAvailableServices().get(service), services.get(service).getVersion());
- }
+
+ // ZK is the only service that is versionable
+ Assert.assertFalse(MapUtils.isEmpty(command.getComponentVersionMap()));
+ Assert.assertEquals(1, command.getComponentVersionMap().size());
+ Assert.assertTrue(command.getComponentVersionMap().containsKey("ZOOKEEPER"));
}
@Test