You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2018/09/07 20:22:37 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-24608]
Package install should handle multiple deployed mpacks (#2271)
This is an automated email from the ASF dual-hosted git repository.
ncole pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 33e86d7 [AMBARI-24608] Package install should handle multiple deployed mpacks (#2271)
33e86d7 is described below
commit 33e86d7ac2e8a670acf5121b321e8426bfa09615
Author: ncole <nc...@hortonworks.com>
AuthorDate: Fri Sep 7 16:22:34 2018 -0400
[AMBARI-24608] Package install should handle multiple deployed mpacks (#2271)
* [AMBARI-24608] Package install should handle multiple deployed mpacks
---
.../UpgradePlanInstallResourceProvider.java | 157 +++++++++++++--------
1 file changed, 96 insertions(+), 61 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradePlanInstallResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradePlanInstallResourceProvider.java
index 8a5addf..56d74bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradePlanInstallResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradePlanInstallResourceProvider.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import org.apache.ambari.annotations.Experimental;
@@ -75,7 +74,6 @@ import org.apache.ambari.server.state.OsSpecific;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang3.math.NumberUtils;
import com.google.common.collect.ImmutableMap;
@@ -97,7 +95,6 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
protected static final String UPGRADE_PLAN_INSTALL_ID = UPGRADE_PLAN_INSTALL + "id";
protected static final String UPGRADE_PLAN_INSTALL_CLUSTER_NAME = UPGRADE_PLAN_INSTALL + "cluster_name";
protected static final String UPGRADE_PLAN_INSTALL_SUCCESS_FACTOR = UPGRADE_PLAN_INSTALL + "success_factor";
- protected static final String UPGRADE_PLAN_INSTALL_FORCE = UPGRADE_PLAN_INSTALL + "force";
private static final Map<Resource.Type, String> KEY_PROPERTY_IDS = ImmutableMap.<Resource.Type, String>builder()
.put(Resource.Type.UpgradePlanInstall, UPGRADE_PLAN_INSTALL_ID)
@@ -230,12 +227,6 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
throw new IllegalArgumentException(String.format("Upgrade plan %s was not found", upgradePlanId));
}
- boolean forceAll = false;
- if (propertyMap.containsKey(UPGRADE_PLAN_INSTALL_FORCE)) {
- String forceAllString = propertyMap.get(UPGRADE_PLAN_INSTALL_FORCE).toString();
- forceAll = BooleanUtils.toBoolean(forceAllString);
- }
-
Float successFactor = DEFAULT_SUCCESS_FACTOR;
if (propertyMap.containsKey(UPGRADE_PLAN_INSTALL_SUCCESS_FACTOR)) {
String successFactorString = propertyMap.get(UPGRADE_PLAN_INSTALL_SUCCESS_FACTOR).toString();
@@ -244,7 +235,7 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
RequestStageContainer installRequest = null;
try {
- installRequest = createOrchestration(cluster, upgradePlan, successFactor, forceAll);
+ installRequest = createOrchestration(cluster, upgradePlan, successFactor);
} catch (AmbariException e) {
throw new IllegalArgumentException(e);
}
@@ -256,16 +247,17 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
@Transactional(rollbackOn= {AmbariException.class, RuntimeException.class})
RequestStageContainer createOrchestration(Cluster cluster, UpgradePlanEntity upgradePlan,
- float successFactor, boolean forceInstall) throws AmbariException, SystemException {
+ float successFactor) throws AmbariException, SystemException {
// TODO - API calls (elsewhere) that make checks for compatible mpacks and selected SGs
// for each plan detail, the mpack and selected SG has has already been
// predetermined via compatibility
- // !!! a single host may get more than one mpack to install. those are different
- // !!! host name -> set of mpack package names
- Map<HostEntity, Set<MpackInstallDetail>> details = new HashMap<>();
+ // !!! an mpack has to be installed across multiple hosts. because of how
+ // repositories are used for the request, each mpack's hosts must be in their
+ // own stage
+ Map<MpackInstallDetail, Set<HostEntity>> details = new HashMap<>();
for (UpgradePlanDetailEntity upgradePlanEntity : upgradePlan.getDetails()) {
long serviceGroupId = upgradePlanEntity.getServiceGroupId();
@@ -288,14 +280,14 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
throw new SystemException("Cannot install upgrade plan as the current mpack is not installed.");
}
+ Set<OsSpecific.Package> packages = new HashSet<>();
+ Set<HostEntity> hosts = new HashSet<>();
mpackHosts.forEach(mpackHostStateEntity ->
{
Host host = cluster.getHost(mpackHostStateEntity.getHostName());
HostEntity hostEntity = mpackHostStateEntity.getHostEntity();
String osFamily = host.getOsFamily();
- List<OsSpecific.Package> packages = new ArrayList<>();
-
OsSpecific anyPackages = targetStack.getOsSpecifics().get(AmbariMetaInfo.ANY_OS);
OsSpecific familyPackages = targetStack.getOsSpecificsSafe().get(osFamily);
// !!! TODO get service specific package names
@@ -303,28 +295,63 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
Arrays.stream(new OsSpecific[] {anyPackages, familyPackages})
.filter(osSpecific -> null != osSpecific && CollectionUtils.isNotEmpty(osSpecific.getPackages()))
.forEach(osSpecific -> {
- packages.addAll(osSpecific.getPackages());
- });
+ packages.addAll(osSpecific.getPackages()); });
- if (!details.containsKey(hostEntity)) {
- details.put(hostEntity, new HashSet<>());
- }
+ hosts.add(hostEntity);
- MpackInstallDetail installDetail = new MpackInstallDetail(targetMpackEntity, packages);
- installDetail.serviceGroupName = serviceGroup.getServiceGroupName();
- details.get(hostEntity).add(installDetail);
});
+
+ MpackInstallDetail installDetail = new MpackInstallDetail(targetMpackEntity, packages);
+ installDetail.serviceGroupName = serviceGroup.getServiceGroupName();
+ details.put(installDetail, hosts);
}
- // !!! at this point we have a map of hosts and the installations that each one needs to run.
+ // !!! at this point we have a map of detail and the hosts they should run on
RequestStageContainer stageContainer = createRequest();
+ details.forEach((installDetail, hosts) -> {
+ try {
+ List<Stage> stages = buildStages(cluster, stageContainer, installDetail,
+ hosts, successFactor);
+
+ stageContainer.addStages(stages);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(e);
+ }
+ });
+
+ stageContainer.persist();
- Iterator<Entry<HostEntity, Set<MpackInstallDetail>>> hostMapIterator = details.entrySet().iterator();
+ return stageContainer;
+ }
+
+ /**
+ * Creates the stages for an mpack install across the cluster.
+ * @param cluster
+ * the cluster
+ * @param stageContainer
+ * the stage container for making stages
+ * @param installDetail
+ * the installation detail instance
+ * @param hosts
+ * the set of hosts for the install detail
+ * @param successFactor
+ * for the stage
+ * @return
+ * the list of stages
+ * @throws AmbariException
+ * @throws SystemException
+ */
+ private List<Stage> buildStages(Cluster cluster,
+ RequestStageContainer stageContainer, MpackInstallDetail installDetail,
+ Set<HostEntity> hosts, float successFactor)
+ throws AmbariException, SystemException {
int maxTasks = s_configuration.getAgentPackageParallelCommandsLimit();
- int hostCount = details.size();
+ int hostCount = hosts.size();
int batchCount = (int) (Math.ceil((double) hostCount / maxTasks));
+ String mpackName = installDetail.mpackEntity.getMpackName();
+ String mpackVersion = installDetail.mpackEntity.getMpackVersion();
long stageId = stageContainer.getLastStageId() + 1;
@@ -353,56 +380,61 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
// add the stage that was just created
stages.add(stage);
+ Iterator<HostEntity> hostIterator = hosts.iterator();
+
// Populate with commands for host
- for (int i = 0; i < maxTasks && hostMapIterator.hasNext(); i++) {
- Entry<HostEntity, Set<MpackInstallDetail>> entry = hostMapIterator.next();
- HostEntity host = entry.getKey();
+ for (int i = 0; i < maxTasks && hostIterator.hasNext(); i++) {
+
+ HostEntity host = hostIterator.next();
// add host to this stage
RequestResourceFilter filter = new RequestResourceFilter(null, null, null,
Collections.singletonList(host.getHostName()));
- List<OsSpecific.Package> packages = new ArrayList<>();
- entry.getValue().forEach(d -> { packages.addAll(d.mpackPackages); });
+ Set<OsSpecific.Package> packages = installDetail.mpackPackages;
Map<String, String> roleParams = ImmutableMap.<String, String>builder()
.put(KeyNames.PACKAGE_LIST, s_gson.toJson(packages))
.build();
- // !!! this loop PROBABLY won't work out, but leave for now to get things going
- for (MpackInstallDetail mpackDetail : entry.getValue()) {
- ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
- MPACK_PACKAGES_ACTION, Collections.singletonList(filter), roleParams);
-
- actionContext.setTimeout(Short.valueOf(s_configuration.getDefaultAgentTaskTimeout(true)));
- actionContext.setExpectedServiceGroupName(mpackDetail.serviceGroupName);
-
- Host h = cluster.getHost(host.getHostName());
- Mpack mpack = getManagementController().getAmbariMetaInfo().getMpack(mpackDetail.mpackId);
- RepoOsEntity repoOsEntity = s_repoHelper.getOSEntityForHost(mpackDetail.mpackEntity, h);
-
- // this isn't being placed correctly elsewhere
- actionContext.addVisitor(command -> {
- try {
- command.setClusterSettings(cluster.getClusterSettingsNameValueMap());
- } catch (AmbariException e) {
- LOG.warn("Could not set cluster settings on the command", e);
- }
- });
+ ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
+ MPACK_PACKAGES_ACTION, Collections.singletonList(filter), roleParams);
- s_repoHelper.addCommandRepositoryToContext(actionContext, mpack, repoOsEntity);
+ actionContext.setTimeout(Short.valueOf(s_configuration.getDefaultAgentTaskTimeout(true)));
+ actionContext.setExpectedServiceGroupName(installDetail.serviceGroupName);
+
+ Host h = cluster.getHost(host.getHostName());
+ Mpack mpack = getManagementController().getAmbariMetaInfo().getMpack(installDetail.mpackId);
+ RepoOsEntity repoOsEntity = s_repoHelper.getOSEntityForHost(installDetail.mpackEntity, h);
+
+ // this isn't being placed correctly elsewhere
+ actionContext.addVisitor(command -> {
+ try {
+ command.setClusterSettings(cluster.getClusterSettingsNameValueMap());
+ } catch (AmbariException e) {
+ LOG.warn("Could not set cluster settings on the command", e);
+ }
+ });
+
+ s_repoHelper.addCommandRepositoryToContext(actionContext, mpack, repoOsEntity);
+
+ s_actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, null);
+
+ // need to set meaningful text on the command
+ stage.getHostRoleCommands().values().forEach(map -> {
+ map.values().forEach(hrc -> {
+ hrc.setCommandDetail(String.format("Install %s %s", mpackName, mpackVersion));
+ });
+ });
- s_actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, null);
- }
}
}
- stageContainer.addStages(stages);
- stageContainer.persist();
-
- return stageContainer;
+ return stages;
}
+
+
@Override
public RequestStatus deleteResourcesAuthorized(Request request, Predicate predicate)
throws SystemException, UnsupportedPropertyException,
@@ -425,13 +457,16 @@ public class UpgradePlanInstallResourceProvider extends AbstractControllerResour
return requestStages;
}
+ /**
+ * Each mpack holds its own information regarding what needs to be installed
+ */
private static class MpackInstallDetail {
private MpackEntity mpackEntity;
private long mpackId = -1L;
- private List<OsSpecific.Package> mpackPackages = new ArrayList<>();
+ private Set<OsSpecific.Package> mpackPackages = new HashSet<>();
private String serviceGroupName;
- private MpackInstallDetail(MpackEntity entity, List<OsSpecific.Package> packages) {
+ private MpackInstallDetail(MpackEntity entity, Set<OsSpecific.Package> packages) {
mpackEntity = entity;
mpackId = entity.getId();
mpackPackages = packages;