You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2018/05/18 17:23:45 UTC
[ambari] branch trunk updated: [AMBARI-23852] - Provide a Framework
For Adding A Component During Upgrade (#1321)
This is an automated email from the ASF dual-hosted git repository.
jonathanhurley pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 02146233 [AMBARI-23852] - Provide a Framework For Adding A Component During Upgrade (#1321)
02146233 is described below
commit 021462339cc4eb4b6c49e7dfbdba29ce77f94a6d
Author: Jonathan Hurley <jo...@apache.org>
AuthorDate: Fri May 18 13:23:40 2018 -0400
[AMBARI-23852] - Provide a Framework For Adding A Component During Upgrade (#1321)
---
.../server/controller/ActionExecutionContext.java | 39 ++++++
.../AmbariCustomCommandExecutionHelper.java | 33 ++++-
.../internal/UpgradeResourceProvider.java | 147 ++++++++++-----------
.../server/serveraction/AbstractServerAction.java | 11 +-
.../serveraction/upgrades/AddComponentAction.java | 140 ++++++++++++++++++++
.../upgrades/ComponentVersionCheckAction.java | 1 +
.../ambari/server/stack/MasterHostResolver.java | 44 ++++++
.../apache/ambari/server/state/UpgradeHelper.java | 34 ++++-
.../ambari/server/state/stack/UpgradePack.java | 54 ++++++--
.../state/stack/upgrade/AddComponentTask.java | 138 +++++++++++++++++++
.../state/stack/upgrade/ClusterGrouping.java | 5 +-
.../server/state/stack/upgrade/ConfigureTask.java | 4 +-
.../server/state/stack/upgrade/Grouping.java | 2 +
.../server/state/stack/upgrade/StageWrapper.java | 4 +-
.../ambari/server/state/stack/upgrade/Task.java | 48 ++++++-
ambari-server/src/main/resources/upgrade-pack.xsd | 13 ++
.../server/api/services/AmbariMetaInfoTest.java | 7 +-
.../AmbariCustomCommandExecutionHelperTest.java | 49 +++++++
.../ambari/server/state/UpgradeHelperTest.java | 24 +++-
.../stacks/HDP/2.0.6/services/YARN/metainfo.xml | 14 ++
20 files changed, 694 insertions(+), 117 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
index e94defc..b677e7f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
@@ -46,6 +46,13 @@ public class ActionExecutionContext {
private boolean allowRetry = false;
private RepositoryVersionEntity repositoryVersion;
+ /**
+ * If {@code true}, instructs Ambari not to worry about whether or not the
+ * command is valid. This is used in cases where we might have to schedule
+ * commands ahead of time for components which are not yet installed.
+ */
+ private boolean isFutureCommand = false;
+
private List<ExecutionCommandVisitor> m_visitors = new ArrayList<>();
/**
@@ -270,4 +277,36 @@ public class ActionExecutionContext {
void visit(ExecutionCommand command);
}
+ /**
+ * Gets whether Ambari should skip all kinds of command verifications while
+ * scheduling since this command runs in the future and might not be
+ * considered "valid".
+ * <p/>
+ * A use case for this would be during an upgrade where trying to schedule
+ * commands for a component which has yet to be added to the cluster (since
+ * it's added as part of the upgrade).
+ *
+ * @return {@code true} if the command is scheduled to run in the future.
+ */
+ public boolean isFutureCommand() {
+ return isFutureCommand;
+ }
+
+ /**
+ * Sets whether Ambari should skip all kinds of command verifications while
+ * scheduling since this command runs in the future and might not be
+ * considered "valid".
+ * <p/>
+ * A use case for this would be during an upgrade where trying to schedule
+ * commands for a component which has yet to be added to the cluster (since
+ * it's added as part of the upgrade).
+ *
+ * @param isFutureCommand
+ * {@code true} to have Ambari skip verification of things like
+ * component hosts while scheduling commands.
+ */
+ public void setIsFutureCommand(boolean isFutureCommand) {
+ this.isFutureCommand = isFutureCommand;
+ }
+
}
\ No newline at end of file
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 0d3cd9d..982f627 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -200,6 +200,11 @@ public class AmbariCustomCommandExecutionHelper {
private Boolean isValidCustomCommand(ActionExecutionContext
actionExecutionContext, RequestResourceFilter resourceFilter)
throws AmbariException {
+
+ if (actionExecutionContext.isFutureCommand()) {
+ return true;
+ }
+
String clusterName = actionExecutionContext.getClusterName();
String serviceName = resourceFilter.getServiceName();
String componentName = resourceFilter.getComponentName();
@@ -271,15 +276,19 @@ public class AmbariCustomCommandExecutionHelper {
// Filter hosts that are in MS
Set<String> ignoredHosts = maintenanceStateHelper.filterHostsInMaintenanceState(
- candidateHosts, new MaintenanceStateHelper.HostPredicate() {
- @Override
- public boolean shouldHostBeRemoved(final String hostname)
- throws AmbariException {
- return !maintenanceStateHelper.isOperationAllowed(
- cluster, actionExecutionContext.getOperationLevel(),
- resourceFilter, serviceName, componentName, hostname);
+ candidateHosts, new MaintenanceStateHelper.HostPredicate() {
+ @Override
+ public boolean shouldHostBeRemoved(final String hostname)
+ throws AmbariException {
+ if (actionExecutionContext.isFutureCommand()) {
+ return false;
}
+
+ return !maintenanceStateHelper.isOperationAllowed(
+ cluster, actionExecutionContext.getOperationLevel(),
+ resourceFilter, serviceName, componentName, hostname);
}
+ }
);
// Filter unhealthy hosts
@@ -309,7 +318,12 @@ public class AmbariCustomCommandExecutionHelper {
}
Service service = cluster.getService(serviceName);
+
+ // grab the stack ID from the service first, and use the context's if it's set
StackId stackId = service.getDesiredStackId();
+ if (null != actionExecutionContext.getRepositoryVersion()) {
+ stackId = actionExecutionContext.getRepositoryVersion().getStackId();
+ }
AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo();
ServiceInfo serviceInfo = ambariMetaInfo.getService(service);
@@ -494,6 +508,11 @@ public class AmbariCustomCommandExecutionHelper {
execCmd.setCommandParams(commandParams);
execCmd.setRoleParams(roleParams);
+ // skip anything else
+ if (actionExecutionContext.isFutureCommand()) {
+ continue;
+ }
+
// perform any server side command related logic - eg - set desired states on restart
applyCustomCommandBackendLogic(cluster, serviceName, componentName, commandName, hostName);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index bbb9de4..52425b4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -17,9 +17,6 @@
*/
package org.apache.ambari.server.controller.internal;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
-
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -86,15 +83,14 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
-import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
-import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.UpgradeContextFactory;
import org.apache.ambari.server.state.UpgradeHelper;
import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder;
import org.apache.ambari.server.state.stack.ConfigUpgradePack;
import org.apache.ambari.server.state.stack.UpgradePack;
+import org.apache.ambari.server.state.stack.upgrade.AddComponentTask;
import org.apache.ambari.server.state.stack.upgrade.ConfigureTask;
import org.apache.ambari.server.state.stack.upgrade.CreateAndConfigureTask;
import org.apache.ambari.server.state.stack.upgrade.Direction;
@@ -476,7 +472,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
if (!countTotals.containsKey(status)) {
countTotals.put(status, Integer.valueOf(0));
}
- double countValue = (double) countTotals.get(status);
+ double countValue = countTotals.get(status);
// !!! calculation lifted from CalculatedStatus
switch (status) {
@@ -495,7 +491,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
break;
default:
if (status.isCompletedState()) {
- percent += countValue / (double) totalTasks;
+ percent += countValue / totalTasks;
}
break;
}
@@ -965,37 +961,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
/**
- * Adds the hooks and service folders based on the effective stack ID and the
- * name of the service from the wrapper.
- *
- * @param wrapper
- * the stage wrapper to use when detemrining the service name.
- * @param effectiveStackId
- * the stack ID to use when getting the hooks and service folders.
- * @param commandParams
- * the params to update with the new values
- * @throws AmbariException
- */
- private void applyRepositoryAssociatedParameters(StageWrapper wrapper, StackId effectiveStackId,
- Map<String, String> commandParams) throws AmbariException {
- if (CollectionUtils.isNotEmpty(wrapper.getTasks())
- && wrapper.getTasks().get(0).getService() != null) {
-
- AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
-
- StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(),
- effectiveStackId.getStackVersion());
-
- String serviceName = wrapper.getTasks().get(0).getService();
- ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(),
- effectiveStackId.getStackVersion(), serviceName);
-
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- commandParams.put(HOOKS_FOLDER, s_configuration.getProperty(Configuration.HOOKS_FOLDER));
- }
- }
-
- /**
* Creates an action stage using the {@link #EXECUTE_TASK_ROLE} custom action
* to execute some Python command.
*
@@ -1073,15 +1038,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
RequestResourceFilter filter = new RequestResourceFilter(serviceName, componentName,
new ArrayList<>(wrapper.getHosts()));
- ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
- EXECUTE_TASK_ROLE, Collections.singletonList(filter), params);
-
- // hosts in maintenance mode are excluded from the upgrade
- actionContext.setMaintenanceModeHostExcluded(true);
-
- actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration));
- actionContext.setRetryAllowed(allowRetry);
- actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped());
+ ActionExecutionContext actionContext = buildActionExecutionContext(cluster, context,
+ EXECUTE_TASK_ROLE, effectiveRepositoryVersion, Collections.singletonList(filter), params,
+ allowRetry, wrapper.getMaxTimeout(s_configuration));
ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext,
cluster, effectiveRepositoryVersion.getStackId(), null);
@@ -1133,8 +1092,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
List<RequestResourceFilter> filters = new ArrayList<>();
for (TaskWrapper tw : wrapper.getTasks()) {
+ String serviceName = tw.getService();
+ String componentName = tw.getComponent();
+
// add each host to this stage
- filters.add(new RequestResourceFilter(tw.getService(), tw.getComponent(),
+ filters.add(
+ new RequestResourceFilter(serviceName, componentName,
new ArrayList<>(tw.getHosts())));
}
@@ -1156,14 +1119,13 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
- ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
- function, filters, commandParams);
- actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration));
- actionContext.setRetryAllowed(allowRetry);
- actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped());
+ ActionExecutionContext actionContext = buildActionExecutionContext(cluster, context, function,
+ effectiveRepositoryVersion, filters, commandParams, allowRetry,
+ wrapper.getMaxTimeout(s_configuration));
- // hosts in maintenance mode are excluded from the upgrade
- actionContext.setMaintenanceModeHostExcluded(true);
+ // commands created here might be for future components which have not been
+ // added to the cluster yet
+ actionContext.setIsFutureCommand(true);
ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext,
cluster, effectiveRepositoryVersion.getStackId(), null);
@@ -1215,16 +1177,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
- ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
- "SERVICE_CHECK", filters, commandParams);
-
- actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration));
- actionContext.setRetryAllowed(allowRetry);
- actionContext.setAutoSkipFailures(context.isServiceCheckFailureAutoSkipped());
-
- // hosts in maintenance mode are excluded from the upgrade and should not be
- // candidates for service checks
- actionContext.setMaintenanceModeHostExcluded(true);
+ ActionExecutionContext actionContext = buildActionExecutionContext(cluster, context,
+ "SERVICE_CHECK", effectiveRepositoryVersion, filters, commandParams, allowRetry,
+ wrapper.getMaxTimeout(s_configuration));
ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext,
cluster, effectiveRepositoryVersion.getStackId(), null);
@@ -1383,6 +1338,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
break;
}
+ case ADD_COMPONENT: {
+ AddComponentTask addComponentTask = (AddComponentTask) task;
+ String serializedTask = addComponentTask.toJson();
+ commandParams.put(AddComponentTask.PARAMETER_SERIALIZED_ADD_COMPONENT_TASK, serializedTask);
+ break;
+ }
default:
break;
}
@@ -1391,16 +1352,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
return false;
}
- ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
- Role.AMBARI_SERVER_ACTION.toString(), Collections.emptyList(),
- commandParams);
-
- actionContext.setTimeout(Short.valueOf((short) -1));
- actionContext.setRetryAllowed(group.allowRetry);
- actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped());
-
- // hosts in maintenance mode are excluded from the upgrade
- actionContext.setMaintenanceModeHostExcluded(true);
+ ActionExecutionContext actionContext = buildActionExecutionContext(cluster, context,
+ Role.AMBARI_SERVER_ACTION.toString(), effectiveRepositoryVersion, Collections.emptyList(),
+ commandParams, group.allowRetry, Short.valueOf((short) -1));
ExecuteCommandJson jsons = s_commandExecutionHelper.get().getCommandJson(actionContext,
cluster, context.getRepositoryVersion().getStackId(), null);
@@ -1596,6 +1550,49 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
/**
+ * Constructs an {@link ActionExecutionContext}, setting common parameters for
+ * all types of commands.
+ *
+ * @param cluster
+ * the cluster
+ * @param context
+ * the upgrade context
+ * @param role
+ * the role for the command
+ * @param repositoryVersion
+ * the repository version which will be used mostly for the stack ID
+ * when building the command and resolving stack-based properties
+ * (like hooks folders)
+ * @param resourceFilters
+ * the filters for where the request will run
+ * @param commandParams
+ * the command parameter map
+ * @param allowRetry
+ * {@code true} to allow retry of the command
+ * @param timeout
+ * the timeout for the command.
+ * @return the {@link ActionExecutionContext}.
+ */
+ private ActionExecutionContext buildActionExecutionContext(Cluster cluster,
+ UpgradeContext context, String role, RepositoryVersionEntity repositoryVersion,
+ List<RequestResourceFilter> resourceFilters, Map<String, String> commandParams,
+ boolean allowRetry, short timeout) {
+
+ ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
+ role, resourceFilters, commandParams);
+
+ actionContext.setRepositoryVersion(repositoryVersion);
+ actionContext.setTimeout(timeout);
+ actionContext.setRetryAllowed(allowRetry);
+ actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped());
+
+ // hosts in maintenance mode are excluded from the upgrade
+ actionContext.setMaintenanceModeHostExcluded(true);
+
+ return actionContext;
+ }
+
+ /**
* Builds the correct {@link ConfigUpgradePack} based on the upgrade and
* source stack.
* <ul>
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/AbstractServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/AbstractServerAction.java
index c13eb5b..7d5a847 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/AbstractServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/AbstractServerAction.java
@@ -31,6 +31,7 @@ import org.apache.ambari.server.audit.AuditLogger;
import org.apache.ambari.server.audit.event.AuditEvent;
import org.apache.ambari.server.utils.StageUtils;
+import com.google.gson.Gson;
import com.google.inject.Inject;
/**
@@ -59,9 +60,15 @@ public abstract class AbstractServerAction implements ServerAction {
@Inject
private AuditLogger auditLogger;
+ /**
+ * Used to deserialized JSON.
+ */
+ @Inject
+ protected Gson gson;
+
@Override
public ExecutionCommand getExecutionCommand() {
- return this.executionCommand;
+ return executionCommand;
}
@Override
@@ -71,7 +78,7 @@ public abstract class AbstractServerAction implements ServerAction {
@Override
public HostRoleCommand getHostRoleCommand() {
- return this.hostRoleCommand;
+ return hostRoleCommand;
}
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java
new file mode 100644
index 0000000..349f54a
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddComponentAction.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.ServiceComponentNotFoundException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.stack.MasterHostResolver;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.State;
+import org.apache.ambari.server.state.UpgradeContext;
+import org.apache.ambari.server.state.stack.upgrade.AddComponentTask;
+
+/**
+ * The {@link AddComponentAction} is used to add a component during an upgrade.
+ */
+public class AddComponentAction extends AbstractUpgradeServerAction {
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+ Map<String, String> commandParameters = getCommandParameters();
+ if (null == commandParameters || commandParameters.isEmpty()) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", "",
+ "Unable to add a new component to the cluster as there is no information on what to add.");
+ }
+
+ String clusterName = commandParameters.get("clusterName");
+ Cluster cluster = getClusters().getCluster(clusterName);
+ UpgradeContext upgradeContext = getUpgradeContext(cluster);
+
+ // guard against downgrade until there is such a thing as removal of a
+ // component on downgrade
+ if (upgradeContext.isDowngradeAllowed() || upgradeContext.isPatchRevert()) {
+ return createCommandReport(0, HostRoleStatus.SKIPPED_FAILED, "{}", "",
+ "Unable to add a component during an upgrade which can be downgraded.");
+ }
+
+ String serializedJson = commandParameters.get(
+ AddComponentTask.PARAMETER_SERIALIZED_ADD_COMPONENT_TASK);
+
+ AddComponentTask task = gson.fromJson(serializedJson, AddComponentTask.class);
+
+ // build the list of candidate hosts
+ Collection<Host> candidates = MasterHostResolver.getCandidateHosts(cluster, task.hosts,
+ task.hostService, task.hostComponent);
+
+ if (candidates.isEmpty()) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", String.format(
+ "Unable to add a new component to the cluster as there are no hosts which contain %s's %s",
+ task.service, task.component));
+ }
+
+ Service service = cluster.getService(task.service);
+ if (null == service) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", "",
+ String.format("Unable to add %s since %s is not installed in this cluster.",
+ task.component, task.service));
+ }
+
+ // create the component if it doesn't exist in the service yet
+ ServiceComponent serviceComponent;
+ try {
+ serviceComponent = service.getServiceComponent(task.component);
+ } catch( ServiceComponentNotFoundException scnfe ) {
+ serviceComponent = service.addServiceComponent(task.component);
+ }
+
+ StringBuilder buffer = new StringBuilder(String.format(
+ "Successfully added %s's %s to the cluster", task.service, task.component)).append(
+ System.lineSeparator());
+
+ Map<String, ServiceComponentHost> existingSCHs = serviceComponent.getServiceComponentHosts();
+
+ for (Host host : candidates) {
+ if (existingSCHs.containsKey(host.getHostName())) {
+ buffer.append(" ")
+ .append(host.getHostName())
+ .append(": ")
+ .append("Already Installed")
+ .append(System.lineSeparator());
+
+ continue;
+ }
+
+ ServiceComponentHost sch = serviceComponent.addServiceComponentHost(host.getHostName());
+ sch.setDesiredState(State.INSTALLED);
+ sch.setState(State.INSTALLED);
+
+ buffer.append(" ")
+ .append(host.getHostName())
+ .append(": ")
+ .append("Installed")
+ .append(System.lineSeparator());
+ }
+
+ Set<String> sortedHosts = candidates.stream().map(host -> host.getHostName()).collect(
+ Collectors.toCollection(() -> new TreeSet<>()));
+
+ Map<String, Object> structureOutMap = new LinkedHashMap<>();
+ structureOutMap.put("service", task.service);
+ structureOutMap.put("component", task.component);
+ structureOutMap.put("hosts", sortedHosts);
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, gson.toJson(structureOutMap),
+ buffer.toString(), "");
+ }
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckAction.java
index f72637e..7485c7a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckAction.java
@@ -68,6 +68,7 @@ public class ComponentVersionCheckAction extends FinalizeUpgradeAction {
private String getErrors(StringBuilder outSB, StringBuilder errSB, Set<InfoTuple> errors) {
errSB.append("Finalization will not be able to completed because of the following version inconsistencies:");
+ errSB.append(System.lineSeparator());
Set<String> hosts = new TreeSet<>();
Map<String, JsonArray> hostDetails = new HashMap<>();
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
index 1b5fbb9..e749195 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
@@ -22,6 +22,7 @@ import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.text.MessageFormat;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -29,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.ambari.metrics.sink.relocated.curator.shaded.com.google.common.collect.Lists;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
@@ -40,6 +42,7 @@ import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.UpgradeState;
import org.apache.ambari.server.state.stack.upgrade.Direction;
+import org.apache.ambari.server.state.stack.upgrade.ExecuteHostType;
import org.apache.ambari.server.utils.HTTPUtils;
import org.apache.ambari.server.utils.HostAndPort;
import org.apache.ambari.server.utils.StageUtils;
@@ -161,6 +164,47 @@ public class MasterHostResolver {
}
/**
+ * Gets hosts which match the supplied criteria.
+ *
+ * @param cluster
+ * @param executeHostType
+ * @param serviceName
+ * @param componentName
+ * @return
+ */
+ public static Collection<Host> getCandidateHosts(Cluster cluster, ExecuteHostType executeHostType,
+ String serviceName, String componentName) {
+ Collection<Host> candidates = cluster.getHosts();
+ if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(componentName)) {
+ List<ServiceComponentHost> schs = cluster.getServiceComponentHosts(serviceName,componentName);
+ candidates = schs.stream().map(sch -> sch.getHost()).collect(Collectors.toList());
+ }
+
+ if (candidates.isEmpty()) {
+ return candidates;
+ }
+
+ // figure out where to add the new component
+ List<Host> hosts = Lists.newArrayList();
+ switch (executeHostType) {
+ case ALL:
+ hosts.addAll(candidates);
+ break;
+ case FIRST:
+ hosts.add(candidates.iterator().next());
+ break;
+ case MASTER:
+ hosts.add(candidates.iterator().next());
+ break;
+ case ANY:
+ hosts.add(candidates.iterator().next());
+ break;
+ }
+
+ return candidates;
+ }
+
+ /**
* Filters the supplied list of hosts in the following ways:
* <ul>
* <li>Compares the versions of a HostComponent to the version for the
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 1550590..c0b6d21 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
@@ -64,6 +65,7 @@ import org.apache.ambari.server.stack.HostsType;
import org.apache.ambari.server.stack.MasterHostResolver;
import org.apache.ambari.server.state.stack.UpgradePack;
import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
+import org.apache.ambari.server.state.stack.upgrade.AddComponentTask;
import org.apache.ambari.server.state.stack.upgrade.Direction;
import org.apache.ambari.server.state.stack.upgrade.Grouping;
import org.apache.ambari.server.state.stack.upgrade.ManualTask;
@@ -309,6 +311,7 @@ public class UpgradeHelper {
Cluster cluster = context.getCluster();
MasterHostResolver mhr = context.getResolver();
+ Map<String, AddComponentTask> addedComponentsDuringUpgrade = upgradePack.getAddComponentTasks();
// Note, only a Rolling Upgrade uses processing tasks.
Map<String, Map<String, ProcessingComponent>> allTasks = upgradePack.getTasks();
@@ -382,16 +385,36 @@ public class UpgradeHelper {
for (String component : service.components) {
// Rolling Upgrade has exactly one task for a Component.
+ // NonRolling Upgrade has several tasks for the same component, since it must first call Stop, perform several
+ // other tasks, and then Start on that Component.
+
if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.get(service.serviceName).containsKey(component)) {
continue;
}
- // NonRolling Upgrade has several tasks for the same component, since it must first call Stop, perform several
- // other tasks, and then Start on that Component.
-
HostsType hostsType = mhr.getMasterAndHosts(service.serviceName, component);
if (null == hostsType) {
- continue;
+ // a null hosts type usually means that the component is not
+ // installed in the cluster - but it's possible that it's going to
+ // be added as part of the upgrade. If this is the case, then we
+ // need to schedule tasks assuming the add works
+ String id = service.serviceName + "/" + component;
+ if (addedComponentsDuringUpgrade.containsKey(id)) {
+ AddComponentTask task = addedComponentsDuringUpgrade.get(id);
+ Collection<Host> candidateHosts = MasterHostResolver.getCandidateHosts(cluster,
+ task.hosts, task.hostService, task.hostComponent);
+
+ if (!candidateHosts.isEmpty()) {
+ hostsType = HostsType.normal(
+ candidateHosts.stream().map(host -> host.getHostName()).collect(
+ Collectors.toCollection(LinkedHashSet::new)));
+ }
+ }
+
+ // if we still have no hosts, then truly skip this component
+ if (null == hostsType) {
+ continue;
+ }
}
if (!hostsType.unhealthy.isEmpty()) {
@@ -804,7 +827,8 @@ public class UpgradeHelper {
* @param component the component name
*/
private void setDisplayNames(UpgradeContext context, String service, String component) {
- StackId stackId = context.getCluster().getDesiredStackVersion();
+ StackId stackId = context.getRepositoryVersion().getStackId();
+
try {
ServiceInfo serviceInfo = m_ambariMetaInfoProvider.get().getService(stackId.getStackName(),
stackId.getStackVersion(), service);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
index 0ce4181..35c081a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
@@ -36,13 +36,16 @@ import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlValue;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.state.stack.upgrade.AddComponentTask;
import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping;
+import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping.ExecuteStage;
import org.apache.ambari.server.state.stack.upgrade.ConfigureTask;
import org.apache.ambari.server.state.stack.upgrade.CreateAndConfigureTask;
import org.apache.ambari.server.state.stack.upgrade.Direction;
import org.apache.ambari.server.state.stack.upgrade.Grouping;
import org.apache.ambari.server.state.stack.upgrade.ServiceCheckGrouping;
import org.apache.ambari.server.state.stack.upgrade.Task;
+import org.apache.ambari.server.state.stack.upgrade.Task.Type;
import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
@@ -97,8 +100,8 @@ public class UpgradePack {
* {@code true} to allow downgrade, {@code false} to disable downgrade.
* Tag is optional and can be {@code null}, use {@code isDowngradeAllowed} getter instead.
*/
- @XmlElement(name = "downgrade-allowed", required = false)
- private boolean downgradeAllowed = true;
+ @XmlElement(name = "downgrade-allowed", required = false, defaultValue = "true")
+ private boolean downgradeAllowed;
/**
* {@code true} to automatically skip service check failures. The default is
@@ -107,15 +110,18 @@ public class UpgradePack {
@XmlElement(name = "skip-service-check-failures")
private boolean skipServiceCheckFailures = false;
- @XmlTransient
- private Map<String, List<String>> m_orders = null;
-
/**
* Initialized once by {@link #afterUnmarshal(Unmarshaller, Object)}.
*/
@XmlTransient
private Map<String, Map<String, ProcessingComponent>> m_process = null;
+ /**
+ * A mapping of SERVICE/COMPONENT to any {@link AddComponentTask} instances.
+ */
+ @XmlTransient
+ private final Map<String, AddComponentTask> m_addComponentTasks = new LinkedHashMap<>();
+
@XmlTransient
private boolean m_resolvedGroups = false;
@@ -152,8 +158,8 @@ public class UpgradePack {
*/
public List<String> getPrerequisiteChecks() {
if (prerequisiteChecks == null) {
- return new ArrayList<String>();
- }
+ return new ArrayList<>();
+ }
return new ArrayList<>(prerequisiteChecks.checks);
}
@@ -164,7 +170,7 @@ public class UpgradePack {
public PrerequisiteCheckConfig getPrerequisiteCheckConfig() {
if (prerequisiteChecks == null) {
return new PrerequisiteCheckConfig();
- }
+ }
return prerequisiteChecks.configuration;
}
@@ -422,6 +428,16 @@ public class UpgradePack {
}
/**
+ * Gets a mapping of SERVICE/COMPONENT to {@link AddComponentTask} for this
+ * upgrade pack.
+ *
+ * @return
+ */
+ public Map<String, AddComponentTask> getAddComponentTasks() {
+ return m_addComponentTasks;
+ }
+
+ /**
* This method is called after all the properties (except IDREF) are
* unmarshalled for this object, but before this object is set to the parent
* object. This is done automatically by the {@link Unmarshaller}.
@@ -440,6 +456,7 @@ public class UpgradePack {
*/
void afterUnmarshal(Unmarshaller unmarshaller, Object parent) {
initializeProcessingComponentMappings();
+ initializeAddComponentTasks();
}
/**
@@ -474,6 +491,27 @@ public class UpgradePack {
}
/**
+ * Builds a mapping of SERVICE/COMPONENT to {@link AddComponentTask}.
+ */
+ private void initializeAddComponentTasks() {
+ for (Grouping group : groups) {
+ if (ClusterGrouping.class.isInstance(group)) {
+ List<ExecuteStage> executeStages = ((ClusterGrouping) group).executionStages;
+ for (ExecuteStage executeStage : executeStages) {
+ Task task = executeStage.task;
+
+ // keep track of this for later ...
+ if (task.getType() == Type.ADD_COMPONENT) {
+ AddComponentTask addComponentTask = (AddComponentTask) task;
+ m_addComponentTasks.put(addComponentTask.getServiceAndComponentAsString(),
+ addComponentTask);
+ }
+ }
+ }
+ }
+ }
+
+ /**
* @return {@code true} if the upgrade targets any version or stack. Both
* {@link #target} and {@link #targetStack} must equal "*"
*/
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/AddComponentTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/AddComponentTask.java
new file mode 100644
index 0000000..28840e6
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/AddComponentTask.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack.upgrade;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.XmlType;
+
+import org.apache.ambari.server.serveraction.upgrades.AddComponentAction;
+
+import com.google.gson.annotations.Expose;
+
+/**
+ * The {@link AddComponentTask} is used for adding components during an upgrade.
+ * Components which are added via the task will also be scheduled for a restart
+ * if they appear in the upgrade pack as part of a restart group. This is true
+ * even if they do not exist in the cluster yet.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "add_component")
+public class AddComponentTask extends ServerSideActionTask {
+
+ /**
+ * The key which represents this task serialized.
+ */
+ public static final String PARAMETER_SERIALIZED_ADD_COMPONENT_TASK = "add-component-task";
+
+ @Expose
+ @XmlTransient
+ private Task.Type type = Type.ADD_COMPONENT;
+
+ /**
+ * The hosts to run the task on. Default to running on
+ * {@link ExecuteHostType#ANY}.
+ */
+ @Expose
+ @XmlAttribute
+ public ExecuteHostType hosts = ExecuteHostType.ANY;
+
+ /**
+ * The service which owns the component to add.
+ */
+ @Expose
+ @XmlAttribute
+ public String service;
+
+ /**
+ * The component to add.
+ */
+ @Expose
+ @XmlAttribute
+ public String component;
+
+ /**
+ * Specifies the hosts which are valid for adding the new component,
+ * restricted by service.
+ */
+ @Expose
+ @XmlAttribute(name = "host-service")
+ public String hostService;
+
+ /**
+ * Specifies the hosts which are valid for adding teh new component,
+ * restricted by component.
+ */
+ @Expose
+ @XmlAttribute(name = "host-component")
+ public String hostComponent;
+
+ /**
+ * Constructor.
+ *
+ */
+ public AddComponentTask() {
+ implClass = AddComponentAction.class.getName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Task.Type getType() {
+ return type;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public StageWrapper.Type getStageWrapperType() {
+ return StageWrapper.Type.SERVER_SIDE_ACTION;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getActionVerb() {
+ return "Adding";
+ }
+
+ /**
+ * Gets a JSON representation of this task.
+ *
+ * @return a JSON representation of this task.
+ */
+ public String toJson() {
+ return GSON.toJson(this);
+ }
+
+ /**
+ * Gets a string which is comprised of serviceName/componentName
+ *
+ * @return a string which represents this add component task.
+ */
+ public String getServiceAndComponentAsString() {
+ return service + "/" + component;
+ }
+}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
index 76550ba..e013d18 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
@@ -46,7 +46,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Objects;
+import com.google.common.base.MoreObjects;
/**
* Used to represent cluster-based operations.
@@ -124,7 +124,7 @@ public class ClusterGrouping extends Grouping {
*/
@Override
public String toString() {
- return Objects.toStringHelper(this).add("id", id).add("title",
+ return MoreObjects.toStringHelper(this).add("id", id).add("title",
title).omitNullValues().toString();
}
@@ -214,6 +214,7 @@ public class ClusterGrouping extends Grouping {
case MANUAL:
case SERVER_ACTION:
case CONFIGURE:
+ case ADD_COMPONENT:
wrapper = getServerActionStageWrapper(upgradeContext, execution);
break;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index 75b5f59..038cd58 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -111,8 +111,6 @@ public class ConfigureTask extends ServerSideActionTask {
implClass = ConfigureAction.class.getName();
}
- private Task.Type type = Task.Type.CONFIGURE;
-
@XmlAttribute(name = "id")
public String id;
@@ -130,7 +128,7 @@ public class ConfigureTask extends ServerSideActionTask {
*/
@Override
public Type getType() {
- return type;
+ return Task.Type.CONFIGURE;
}
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index 897c643..cc42d65 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -325,8 +325,10 @@ public class Grouping {
private TaskBucket(Task initial) {
switch (initial.getType()) {
case CONFIGURE:
+ case CREATE_AND_CONFIGURE:
case SERVER_ACTION:
case MANUAL:
+ case ADD_COMPONENT:
type = StageWrapper.Type.SERVER_SIDE_ACTION;
break;
case EXECUTE:
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
index 79147aa..db8f849 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
@@ -31,7 +31,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Objects;
+import com.google.common.base.MoreObjects;
import com.google.gson.Gson;
/**
@@ -167,7 +167,7 @@ public class StageWrapper {
*/
@Override
public String toString() {
- return Objects.toStringHelper(this).add("type", type)
+ return MoreObjects.toStringHelper(this).add("type", type)
.add("text",text)
.omitNullValues().toString();
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
index 3426a3a..8b141a8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
@@ -17,20 +17,40 @@
*/
package org.apache.ambari.server.state.stack.upgrade;
+import java.util.EnumSet;
+
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlSeeAlso;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.Expose;
+
/**
* Base class to identify the items that could possibly occur during an upgrade
*/
-@XmlSeeAlso(value={ExecuteTask.class, CreateAndConfigureTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class, StartTask.class, StopTask.class, ServerActionTask.class, ConfigureFunction.class})
+@XmlSeeAlso(
+ value = { ExecuteTask.class, CreateAndConfigureTask.class, ConfigureTask.class,
+ ManualTask.class, RestartTask.class, StartTask.class, StopTask.class,
+ ServerActionTask.class, ConfigureFunction.class, AddComponentTask.class })
public abstract class Task {
/**
+ * A Gson instance to use when doing simple serialization along with
+ * {@link Expose}.
+ */
+ protected final static Gson GSON = new GsonBuilder()
+ .serializeNulls()
+ .setPrettyPrinting()
+ .excludeFieldsWithoutExposeAnnotation()
+ .create();
+
+ /**
* An optional brief description of what this task is doing.
*/
+ @Expose
@XmlElement(name = "summary")
public String summary;
@@ -38,12 +58,14 @@ public abstract class Task {
* Whether the task needs to run sequentially, i.e., on its own stage.
* If false, will be grouped with other tasks.
*/
+ @Expose
@XmlAttribute(name = "sequential")
public boolean isSequential = false;
/**
* The config property to check for timeout.
*/
+ @Expose
@XmlAttribute(name="timeout-config")
public String timeoutConfig = null;
@@ -72,6 +94,7 @@ public abstract class Task {
/**
* The scope for the task
*/
+ @Expose
@XmlElement(name = "scope")
public UpgradeScope scope = UpgradeScope.ANY;
@@ -133,20 +156,37 @@ public abstract class Task {
/**
* Task meant to run against Ambari server.
*/
- SERVER_ACTION;
+ SERVER_ACTION,
+
+ /**
+ * A task which adds new components to the cluster during the upgrade.
+ */
+ ADD_COMPONENT;
+
+ /**
+ * Commands which run on the server.
+ */
+ public static final EnumSet<Type> SERVER_ACTIONS = EnumSet.of(MANUAL, CONFIGURE, SERVER_ACTION,
+ ADD_COMPONENT);
+
+ /**
+ * Commands which are run on agents.
+ */
+ public static final EnumSet<Type> COMMANDS = EnumSet.of(RESTART, START, CONFIGURE_FUNCTION,
+ STOP, SERVICE_CHECK);
/**
* @return {@code true} if the task is manual or automated.
*/
public boolean isServerAction() {
- return this == MANUAL || this == CONFIGURE || this == SERVER_ACTION;
+ return SERVER_ACTIONS.contains(this);
}
/**
* @return {@code true} if the task is a command type (as opposed to an action)
*/
public boolean isCommand() {
- return this == RESTART || this == START || this == CONFIGURE_FUNCTION || this == STOP || this == SERVICE_CHECK;
+ return COMMANDS.contains(this);
}
}
}
diff --git a/ambari-server/src/main/resources/upgrade-pack.xsd b/ambari-server/src/main/resources/upgrade-pack.xsd
index 249db23..c77e29b 100644
--- a/ambari-server/src/main/resources/upgrade-pack.xsd
+++ b/ambari-server/src/main/resources/upgrade-pack.xsd
@@ -361,6 +361,19 @@
</xs:extension>
</xs:complexContent>
</xs:complexType>
+
+ <xs:complexType name="add_component">
+ <xs:complexContent>
+ <xs:extension base="abstract-server-task-type">
+ <xs:sequence />
+ <xs:attribute name="service" type="xs:string" use="required"/>
+ <xs:attribute name="component" type="xs:string" use="required"/>
+ <xs:attribute name="host-service" type="xs:string" use="optional"/>
+ <xs:attribute name="host-component" type="xs:string" use="optional"/>
+ <xs:attribute name="hosts" use="required" type="host-target-type" />
+ </xs:extension>
+ </xs:complexContent>
+ </xs:complexType>
<xs:complexType name="order-type">
<xs:sequence>
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
index a4c5502..426f6b4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
@@ -120,9 +120,6 @@ public class AmbariMetaInfoTest {
private static final String SERVICE_NAME_MAPRED2 = "MAPREDUCE2";
private static final String SERVICE_COMPONENT_NAME = "NAMENODE";
private static final String OS_TYPE = "centos5";
- private static final String HDP_REPO_NAME = "HDP";
- private static final String HDP_REPO_ID = "HDP-2.1.1";
- private static final String HDP_UTILS_REPO_NAME = "HDP-UTILS";
private static final String REPO_ID = "HDP-UTILS-1.1.0.15";
private static final String PROPERTY_NAME = "hbase.regionserver.msginterval";
private static final String SHARED_PROPERTY_NAME = "content";
@@ -621,8 +618,8 @@ public class AmbariMetaInfoTest {
"the extended stack.", deletedService);
Assert.assertNotNull(redefinedService);
// Components
- Assert.assertEquals("YARN service is expected to be defined with 3 active" +
- " components.", 3, redefinedService.getComponents().size());
+ Assert.assertEquals("YARN service is expected to be defined with 4 active" +
+ " components.", 4, redefinedService.getComponents().size());
Assert.assertEquals("TEZ is expected to be a part of extended stack " +
"definition", "TEZ", redefinedService.getClientComponent().getName());
Assert.assertFalse("YARN CLIENT is a deleted component.",
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
index 56036fe..3533a7d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
@@ -695,6 +695,55 @@ public class AmbariCustomCommandExecutionHelperTest {
Assert.assertEquals(2, commandRepo.getRepositories().size());
}
+ /**
+ * Tests that when {@link ActionExecutionContext#isFutureCommand()} is set, invalid
+ * hosts/components are still scheduled.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFutureCommandsPassValidation() throws Exception {
+ // make sure the component isn't added to the cluster yet
+ String serviceName = "YARN";
+ String componentName = "YARN_FOO";
+ Cluster cluster = clusters.getCluster("c1");
+ Service service = cluster.getService(serviceName);
+ try {
+ service.getServiceComponent(componentName);
+ Assert.fail("For this test to work, the YARN_FOO component must not be in the cluster yet");
+ } catch (AmbariException ambariException) {
+ // expected
+ }
+
+ AmbariCustomCommandExecutionHelper ambariCustomCommandExecutionHelper = injector.getInstance(AmbariCustomCommandExecutionHelper.class);
+
+ List<RequestResourceFilter> requestResourceFilter = new ArrayList<RequestResourceFilter>() {{
+ add(new RequestResourceFilter(serviceName, componentName,
+ Collections.singletonList("c1-c6401")));
+ }};
+
+ ActionExecutionContext actionExecutionContext = new ActionExecutionContext("c1", "RESTART", requestResourceFilter);
+ actionExecutionContext.setIsFutureCommand(true);
+
+ Stage stage = EasyMock.niceMock(Stage.class);
+ ExecutionCommandWrapper execCmdWrapper = EasyMock.niceMock(ExecutionCommandWrapper.class);
+ ExecutionCommand execCmd = EasyMock.niceMock(ExecutionCommand.class);
+
+ EasyMock.expect(stage.getClusterName()).andReturn("c1");
+
+ EasyMock.expect(stage.getExecutionCommandWrapper(EasyMock.eq("c1-c6401"), EasyMock.anyString())).andReturn(execCmdWrapper);
+ EasyMock.expect(execCmdWrapper.getExecutionCommand()).andReturn(execCmd);
+ EasyMock.expectLastCall();
+
+ HashSet<String> localComponents = new HashSet<>();
+ EasyMock.expect(execCmd.getLocalComponents()).andReturn(localComponents).anyTimes();
+ EasyMock.replay(configHelper, stage, execCmdWrapper, execCmd);
+
+ ambariCustomCommandExecutionHelper.addExecutionCommandsToStage(actionExecutionContext, stage, new HashMap<>(), null);
+
+ EasyMock.verify(configHelper, stage, execCmdWrapper, execCmd);
+ }
+
private void createClusterFixture(String clusterName, StackId stackId,
String respositoryVersion, String hostPrefix) throws AmbariException, AuthorizationException, NoSuchFieldException, IllegalAccessException {
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 4f8e950..a23c451 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -231,8 +231,12 @@ public class UpgradeHelperTest extends EasyMockSupport {
upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
- ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+ // set the display names of the service and component in the target stack
+ // to make sure that we can correctly render display strings during the
+ // upgrade
+ ServiceInfo si = ambariMetaInfo.getService("HDP", "2.2.0", "ZOOKEEPER");
si.setDisplayName("Zk");
+
ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
ci.setDisplayName("ZooKeeper1 Server2");
@@ -310,8 +314,12 @@ public class UpgradeHelperTest extends EasyMockSupport {
upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
- ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+ // set the display names of the service and component in the target stack
+ // to make sure that we can correctly render display strings during the
+ // upgrade
+ ServiceInfo si = ambariMetaInfo.getService("HDP", "2.2.0", "ZOOKEEPER");
si.setDisplayName("Zk");
+
ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
ci.setDisplayName("ZooKeeper1 Server2");
@@ -367,8 +375,12 @@ public class UpgradeHelperTest extends EasyMockSupport {
upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
- ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+ // set the display names of the service and component in the target stack
+ // to make sure that we can correctly render display strings during the
+ // upgrade
+ ServiceInfo si = ambariMetaInfo.getService("HDP", "2.2.0", "ZOOKEEPER");
si.setDisplayName("Zk");
+
ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
ci.setDisplayName("ZooKeeper1 Server2");
@@ -1255,8 +1267,12 @@ public class UpgradeHelperTest extends EasyMockSupport {
public void testUpgradeOrchestrationFullTask() throws Exception {
Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
- ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER");
+ // set the display names of the service and component in the target stack
+ // to make sure that we can correctly render display strings during the
+ // upgrade
+ ServiceInfo si = ambariMetaInfo.getService("HDP", "2.2.0", "ZOOKEEPER");
si.setDisplayName("Zk");
+
ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER");
ci.setDisplayName("ZooKeeper1 Server2");
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
index 8ce291c..713eb7d 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/metainfo.xml
@@ -71,6 +71,19 @@
<timeout>600</timeout>
</commandScript>
</component>
+
+ <component>
+ <name>YARN_FOO</name>
+ <displayName>Yarn Foo</displayName>
+ <category>SLAVE</category>
+ <cardinality>0+</cardinality>
+ <commandScript>
+ <script>scripts/yarn_foo.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
<component>
<name>YARN_CLIENT</name>
<displayName>YARN Client</displayName>
@@ -83,6 +96,7 @@
<timeout>600</timeout>
</commandScript>
</component>
+
<component>
<name>TEZ</name>
<category>CLIENT</category>
--
To stop receiving notification emails like this one, please contact
jonathanhurley@apache.org.