You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/01/18 15:58:19 UTC
[46/50] [abbrv] ambari git commit: AMBARI-19423. Change
HostStackVersionResourceProvider to be able to install packages on single
host not belonging to any cluster (magyari_sandor)
AMBARI-19423. Change HostStackVersionResourceProvider to be able to install packages on single host not belonging to any cluster (magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f684c2b8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f684c2b8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f684c2b8
Branch: refs/heads/branch-dev-patch-upgrade
Commit: f684c2b874c54108e8a313361a0ee8f856ff7dee
Parents: ed93a56
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Wed Jan 4 13:44:25 2017 +0100
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Wed Jan 18 11:56:40 2017 +0100
----------------------------------------------------------------------
.../controller/AmbariActionExecutionHelper.java | 33 ++++--
.../HostStackVersionResourceProvider.java | 117 +++++++++++++++----
.../HostStackVersionResourceProviderTest.java | 96 +++++++++++++++
3 files changed, 211 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f684c2b8/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 4fa942f..ec0f7d0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -227,17 +227,28 @@ public class AmbariActionExecutionHelper {
|| targetHostType.equals(TargetHostType.MAJORITY);
}
-
/**
* Add tasks to the stage based on the requested action execution
- *
* @param actionContext the context associated with the action
* @param stage stage into which tasks must be inserted
* @param requestParams all request parameters (may be null)
* @throws AmbariException if the task can not be added
*/
public void addExecutionCommandsToStage(final ActionExecutionContext actionContext, Stage stage,
- Map<String, String> requestParams)
+ Map<String, String> requestParams) throws AmbariException {
+ addExecutionCommandsToStage(actionContext, stage, requestParams, true);
+ }
+
+ /**
+ * Add tasks to the stage based on the requested action execution
+ * @param actionContext
+ * @param stage
+ * @param requestParams
+ * @param checkHostIsMemberOfCluster if true AmbariException will be thrown in case host is not member of cluster.
+ * @throws AmbariException
+ */
+ public void addExecutionCommandsToStage(final ActionExecutionContext actionContext, Stage stage,
+ Map<String, String> requestParams, boolean checkHostIsMemberOfCluster)
throws AmbariException {
String actionName = actionContext.getActionName();
@@ -331,13 +342,15 @@ public class AmbariActionExecutionHelper {
+ "actionName=" + actionContext.getActionName());
}
- // Compare specified hosts to available hosts
- if (!resourceFilter.getHostNames().isEmpty() && !candidateHosts.isEmpty()) {
- for (String hostname : resourceFilter.getHostNames()) {
- if (!candidateHosts.contains(hostname)) {
- throw new AmbariException("Request specifies host " + hostname +
- " but it is not a valid host based on the " +
- "target service=" + serviceName + " and component=" + componentName);
+ if (checkHostIsMemberOfCluster) {
+ // Compare specified hosts to available hosts
+ if (!resourceFilter.getHostNames().isEmpty() && !candidateHosts.isEmpty()) {
+ for (String hostname : resourceFilter.getHostNames()) {
+ if (!candidateHosts.contains(hostname)) {
+ throw new AmbariException("Request specifies host " + hostname +
+ " but it is not a valid host based on the " +
+ "target service=" + serviceName + " and component=" + componentName);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f684c2b8/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java
index b8d7381..364a61e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java
@@ -66,6 +66,7 @@ import org.apache.ambari.server.state.ServiceOsSpecific;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.Validate;
@@ -91,6 +92,18 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
protected static final String HOST_STACK_VERSION_REPOSITORIES_PROPERTY_ID = PropertyHelper.getPropertyId("HostStackVersions", "repositories");
protected static final String HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("HostStackVersions", "repository_version");
+ /**
+ * Whether to force creating of install command on a host which is not member of any cluster yet.
+ */
+ protected static final String HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID = PropertyHelper
+ .getPropertyId("HostStackVersions", "force_non_member_install");
+
+ /**
+ * In case of force_non_member_install = true a list of component names must be provided in the request.
+ */
+ protected static final String HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID = PropertyHelper.getPropertyId("HostStackVersions", "components");
+ protected static final String COMPONENT_NAME_PROPERTY_ID = "name";
+
protected static final String INSTALL_PACKAGES_ACTION = "install_packages";
protected static final String INSTALL_PACKAGES_FULL_NAME = "Install version";
@@ -111,7 +124,9 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
HOST_STACK_VERSION_VERSION_PROPERTY_ID,
HOST_STACK_VERSION_STATE_PROPERTY_ID,
HOST_STACK_VERSION_REPOSITORIES_PROPERTY_ID,
- HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID);
+ HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID,
+ HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID,
+ HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID);
private static Map<Type, String> keyPropertyIds = new HashMap<Type, String>() {
{
@@ -261,9 +276,36 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
String.format("The required property %s is not defined", requiredProperty));
}
- String clName = (String) propertyMap.get(HOST_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID);
+ String clName = (String) propertyMap.get (HOST_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID);
hostName = (String) propertyMap.get(HOST_STACK_VERSION_HOST_NAME_PROPERTY_ID);
desiredRepoVersion = (String) propertyMap.get(HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID);
+ stackName = (String) propertyMap.get(HOST_STACK_VERSION_STACK_PROPERTY_ID);
+ stackVersion = (String) propertyMap.get(HOST_STACK_VERSION_VERSION_PROPERTY_ID);
+
+ boolean forceInstallOnNonMemberHost = false;
+ Set<Map<String, String>> componentNames = null;
+ String forceInstallOnNonMemberHostString = (String) propertyMap.get
+ (HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID);
+
+ if (BooleanUtils.toBoolean(forceInstallOnNonMemberHostString)) {
+ forceInstallOnNonMemberHost = true;
+ componentNames = (Set<Map<String, String>>) propertyMap.get(HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID);
+ if (componentNames == null) {
+ throw new IllegalArgumentException("In case " + HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID + " is set to true, the list of " +
+ "components should be specified in request.");
+ }
+ }
+
+ RequestStageContainer req = createInstallPackagesRequest(hostName, desiredRepoVersion, stackName, stackVersion,
+ clName, forceInstallOnNonMemberHost, componentNames);
+ return getRequestStatus(req.getRequestStatusResponse());
+ }
+
+ private RequestStageContainer createInstallPackagesRequest(String hostName, final String desiredRepoVersion,
+ String stackName, String stackVersion, String clName,
+ boolean forceInstallOnNonMemberHost,
+ Set<Map<String, String>> componentNames)
+ throws NoSuchParentResourceException, SystemException {
Host host;
try {
@@ -275,8 +317,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
AmbariManagementController managementController = getManagementController();
AmbariMetaInfo ami = managementController.getAmbariMetaInfo();
- stackName = (String) propertyMap.get(HOST_STACK_VERSION_STACK_PROPERTY_ID);
- stackVersion = (String) propertyMap.get(HOST_STACK_VERSION_VERSION_PROPERTY_ID);
final StackId stackId = new StackId(stackName, stackVersion);
if (!ami.isSupportedStack(stackName, stackVersion)) {
throw new NoSuchParentResourceException(String.format("Stack %s is not supported",
@@ -327,17 +367,19 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
HostVersionEntity hostVersEntity = hostVersionDAO.findByClusterStackVersionAndHost(clName, stackId,
desiredRepoVersion, hostName);
- if (hostVersEntity == null) {
- throw new IllegalArgumentException(String.format(
- "Repo version %s for stack %s is not available for host %s",
- desiredRepoVersion, stackId, hostName));
- }
- if (hostVersEntity.getState() != RepositoryVersionState.INSTALLED &&
- hostVersEntity.getState() != RepositoryVersionState.INSTALL_FAILED &&
- hostVersEntity.getState() != RepositoryVersionState.OUT_OF_SYNC) {
- throw new UnsupportedOperationException(String.format("Repo version %s for stack %s " +
- "for host %s is in %s state. Can not transition to INSTALLING state",
- desiredRepoVersion, stackId, hostName, hostVersEntity.getState().toString()));
+ if (!forceInstallOnNonMemberHost) {
+ if (hostVersEntity == null) {
+ throw new IllegalArgumentException(String.format(
+ "Repo version %s for stack %s is not available for host %s",
+ desiredRepoVersion, stackId, hostName));
+ }
+ if (hostVersEntity.getState() != RepositoryVersionState.INSTALLED &&
+ hostVersEntity.getState() != RepositoryVersionState.INSTALL_FAILED &&
+ hostVersEntity.getState() != RepositoryVersionState.OUT_OF_SYNC) {
+ throw new UnsupportedOperationException(String.format("Repo version %s for stack %s " +
+ "for host %s is in %s state. Can not transition to INSTALLING state",
+ desiredRepoVersion, stackId, hostName, hostVersEntity.getState().toString()));
+ }
}
List<OperatingSystemEntity> operatingSystems = repoVersionEnt.getOperatingSystems();
@@ -357,9 +399,34 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
// For every host at cluster, determine packages for all installed services
List<ServiceOsSpecific.Package> packages = new ArrayList<>();
Set<String> servicesOnHost = new HashSet<>();
- List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName());
- for (ServiceComponentHost component : components) {
- servicesOnHost.add(component.getServiceName());
+
+ if (forceInstallOnNonMemberHost) {
+ for (Map<String, String> componentProperties : componentNames) {
+
+ String componentName = componentProperties.get(COMPONENT_NAME_PROPERTY_ID);
+ if (StringUtils.isEmpty(componentName)) {
+ throw new IllegalArgumentException("Components list contains a component with no 'name' property");
+ }
+
+ String serviceName = null;
+ try {
+ serviceName = ami.getComponentToService(stackName, stackVersion, componentName.trim().toUpperCase());
+ if (serviceName == null) {
+ throw new IllegalArgumentException("Service not found for component : " + componentName);
+ }
+ servicesOnHost.add(serviceName);
+ } catch (AmbariException e) {
+ LOG.error("Service not found for component {}!", componentName, e);
+ throw new IllegalArgumentException("Service not found for component : " + componentName);
+ }
+
+ }
+
+ } else {
+ List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName());
+ for (ServiceComponentHost component : components) {
+ servicesOnHost.add(component.getServiceName());
+ }
}
List<String> blacklistedPackagePrefixes = configuration.getRollingUpgradeSkipPackagesPrefixes();
for (String serviceName : servicesOnHost) {
@@ -409,7 +476,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
params.put(KeyNames.PACKAGE_VERSION, xml.getPackageVersion(osFamily));
}
-
// Create custom action
RequestResourceFilter filter = new RequestResourceFilter(null, null,
Collections.singletonList(hostName));
@@ -452,21 +518,22 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource
req.addStages(Collections.singletonList(stage));
try {
- actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, null);
+ actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, null, !forceInstallOnNonMemberHost);
} catch (AmbariException e) {
throw new SystemException("Can not modify stage", e);
}
try {
- hostVersEntity.setState(RepositoryVersionState.INSTALLING);
- hostVersionDAO.merge(hostVersEntity);
-
- cluster.recalculateClusterVersionState(repoVersionEnt);
+ if (!forceInstallOnNonMemberHost) {
+ hostVersEntity.setState(RepositoryVersionState.INSTALLING);
+ hostVersionDAO.merge(hostVersEntity);
+ cluster.recalculateClusterVersionState(repoVersionEnt);
+ }
req.persist();
} catch (AmbariException e) {
throw new SystemException("Can not persist request", e);
}
- return getRequestStatus(req.getRequestStatusResponse());
+ return req;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f684c2b8/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java
index b7fd051..57b77b4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java
@@ -32,6 +32,7 @@ import static org.easymock.EasyMock.verify;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -292,6 +293,101 @@ public class HostStackVersionResourceProviderTest {
}
@Test
+ public void testCreateResources_on_host_not_belonging_To_any_cluster() throws Exception {
+ StackId stackId = new StackId("HDP", "2.0.1");
+
+ final Host host1 = createNiceMock("host1", Host.class);
+ expect(host1.getHostName()).andReturn("host1").anyTimes();
+ expect(host1.getOsFamily()).andReturn("redhat6").anyTimes();
+ replay(host1);
+ Map<String, Host> hostsForCluster = new HashMap<>();
+
+ ServiceComponentHost sch = createMock(ServiceComponentHost.class);
+
+ final ServiceOsSpecific.Package hivePackage = new ServiceOsSpecific.Package();
+ hivePackage.setName("hive");
+ final ServiceOsSpecific.Package mysqlPackage = new ServiceOsSpecific.Package();
+ mysqlPackage.setName("mysql");
+ mysqlPackage.setSkipUpgrade(Boolean.TRUE);
+ List<ServiceOsSpecific.Package> packages = Arrays.asList(hivePackage, mysqlPackage);
+
+ AbstractControllerResourceProvider.init(resourceProviderFactory);
+
+ Map<String, Map<String, String>> hostConfigTags = new HashMap<>();
+ expect(configHelper.getEffectiveDesiredTags(anyObject(ClusterImpl.class), anyObject(String.class))).andReturn(hostConfigTags);
+
+ expect(managementController.getClusters()).andReturn(clusters).anyTimes();
+ expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
+ expect(managementController.getActionManager()).andReturn(actionManager).anyTimes();
+ expect(managementController.getJdkResourceUrl()).andReturn("/JdkResourceUrl").anyTimes();
+ expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class),
+ anyObject(Map.class), anyObject(String.class))).andReturn(packages).anyTimes();
+
+ expect(resourceProviderFactory.getHostResourceProvider(anyObject(Set.class), anyObject(Map.class),
+ eq(managementController))).andReturn(csvResourceProvider).anyTimes();
+
+ expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster);
+ expect(clusters.getHost(anyObject(String.class))).andReturn(host1);
+ expect(cluster.getHosts()).andReturn(hostsForCluster.values()).atLeastOnce();
+ expect(cluster.getServices()).andReturn(new HashMap<String, Service>()).anyTimes();
+ expect(cluster.getCurrentStackVersion()).andReturn(stackId);
+
+ expect(
+ repositoryVersionDAOMock.findByStackAndVersion(
+ anyObject(StackId.class),
+ anyObject(String.class))).andReturn(repoVersion);
+
+ expect(actionManager.getRequestTasks(anyLong())).andReturn(Collections.<HostRoleCommand>emptyList()).anyTimes();
+
+ StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class));
+ StageUtils.setConfiguration(injector.getInstance(Configuration.class));
+
+ // replay
+ replay(managementController, response, clusters, resourceProviderFactory, csvResourceProvider,
+ cluster, repositoryVersionDAOMock, configHelper, sch, actionManager, hostVersionEntityMock, hostVersionDAOMock);
+
+ ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(
+ type,
+ PropertyHelper.getPropertyIds(type),
+ PropertyHelper.getKeyPropertyIds(type),
+ managementController);
+
+ injector.injectMembers(provider);
+
+ // add the property map to a set for the request. add more maps for multiple creates
+ Set<Map<String, Object>> propertySet = new LinkedHashSet<>();
+
+ Map<String, Object> properties = new LinkedHashMap<>();
+
+ // add properties to the request map
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, "Cluster100");
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID, "2.2.0.1-885");
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_STACK_PROPERTY_ID, "HDP");
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_VERSION_PROPERTY_ID, "2.0.1");
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_HOST_NAME_PROPERTY_ID, "host1");
+ Set<Map<String, String>> components = new HashSet<>();
+ Map<String, String> hiveMetastoreComponent = new HashMap<>();
+ hiveMetastoreComponent.put("name", "HIVE_METASTORE");
+ components.add(hiveMetastoreComponent);
+ Map<String, String> hiveServerstoreComponent = new HashMap<>();
+ hiveServerstoreComponent.put("name", "HIVE_SERVER");
+ components.add(hiveServerstoreComponent);
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID, components);
+ properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID,
+ "true");
+
+ propertySet.add(properties);
+
+ // create the request
+ Request request = PropertyHelper.getCreateRequest(propertySet, null);
+
+ provider.createResources(request);
+
+ // verify
+ verify(managementController, response, clusters);
+ }
+
+ @Test
public void testCreateResources_in_out_of_sync_state() throws Exception {
StackId stackId = new StackId("HDP", "2.0.1");