You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2016/05/19 21:21:31 UTC
[2/2] ambari git commit: AMBARI-15388 - Upgrade XML should be pushed
down as much as possible to the services (Tim Thorpe via jluniya)
AMBARI-15388 - Upgrade XML should be pushed down as much as possible to the services (Tim Thorpe via jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8b42749
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8b42749
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8b42749
Branch: refs/heads/trunk
Commit: f8b427494b36c92f8a36c59d1a7233b5a130f4e7
Parents: 42e69f3
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Thu May 19 14:21:22 2016 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Thu May 19 14:21:22 2016 -0700
----------------------------------------------------------------------
.../server/stack/CommonServiceDirectory.java | 10 +
.../ambari/server/stack/ServiceDirectory.java | 19 ++
.../ambari/server/stack/ServiceModule.java | 4 +
.../ambari/server/stack/StackDirectory.java | 66 ++++--
.../apache/ambari/server/stack/StackModule.java | 233 +++++++++++++++++++
.../server/stack/StackServiceDirectory.java | 30 ++-
.../apache/ambari/server/state/ServiceInfo.java | 14 ++
.../ambari/server/state/stack/UpgradePack.java | 72 ++++++
.../state/stack/upgrade/ClusterGrouping.java | 70 ++++++
.../server/state/stack/upgrade/Grouping.java | 69 ++++++
.../stack/upgrade/ServiceCheckGrouping.java | 78 +++++++
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 6 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 6 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 6 +-
.../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 6 +-
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 6 +-
.../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 6 +-
.../server/stack/StackManagerMiscTest.java | 31 +++
.../server/state/stack/UpgradePackTest.java | 66 ++++++
.../upgrades/HDP/2.2.0/upgrade_test_15388.xml | 94 ++++++++
.../HDP/2.2.0/upgrades/upgrade_test_15388.xml | 232 ++++++++++++++++++
.../HDP/2.2.0/metainfo.xml | 22 ++
.../HDP/2.2.0/repos/hdp.json | 15 ++
.../HDP/2.2.0/repos/repoinfo.xml | 34 +++
.../HDP/2.2.0/repos/version-2.2.0.4-123.xml | 49 ++++
.../HDP/2.2.0/role_command_order.json | 80 +++++++
.../HDP/2.2.0/services/HDFS/metainfo.xml | 203 ++++++++++++++++
.../upgrades/HDP/2.2.0/upgrade_test_15388.xml | 93 ++++++++
.../HDP/2.2.0/upgrades/config-upgrade.xml | 101 ++++++++
.../HDP/2.2.0/upgrades/upgrade_test_15388.xml | 232 ++++++++++++++++++
30 files changed, 1905 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java
index 7f7a49e..cbbdb91 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java
@@ -85,6 +85,16 @@ public class CommonServiceDirectory extends ServiceDirectory {
LOG.debug(String.format("Service package folder %s for common service %s does not exist.",
absPackageDir, serviceId ));
}
+
+ File absUpgradesDir = new File(getAbsolutePath() + File.separator + UPGRADES_FOLDER_NAME);
+ if(absUpgradesDir.isDirectory()) {
+ upgradesDir = absUpgradesDir;
+ LOG.debug(String.format("Service upgrades folder for common service %s has been resolved to %s",
+ serviceId, upgradesDir));
+ } else {
+ LOG.debug(String.format("Service upgrades folder %s for common service %s does not exist.",
+ absUpgradesDir, serviceId ));
+ }
parseMetaInfoFile();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
index 8a7b42b..9ed2c24 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
@@ -70,6 +70,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
protected String packageDir;
/**
+ * upgrades directory path
+ */
+ protected File upgradesDir;
+
+ /**
* service metainfo file object representation
*/
private ServiceMetainfoXml metaInfoXml;
@@ -85,6 +90,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
protected static final String PACKAGE_FOLDER_NAME = "package";
/**
+ * upgrades directory name
+ */
+ protected static final String UPGRADES_FOLDER_NAME = "upgrades";
+
+ /**
* service metainfo file name
*/
private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml";
@@ -148,6 +158,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
}
/**
+ * Obtain the upgrades directory path.
+ *
+ * @return upgrades directory path
+ */
+ public File getUpgradesDir() {
+ return upgradesDir;
+ }
+
+ /**
* Obtain the metrics file.
*
* @return metrics file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
index f781574..17a2a93 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
@@ -133,6 +133,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem
serviceInfo.setWidgetsDescriptorFile(serviceDirectory.getWidgetsDescriptorFile(serviceInfo.getName()));
serviceInfo.setSchemaVersion(AmbariMetaInfo.SCHEMA_VERSION_2);
serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir());
+ serviceInfo.setServiceUpgradesFolder(serviceDirectory.getUpgradesDir());
serviceInfo.setAdvisorFile(serviceDirectory.getAdvisorFile());
serviceInfo.setAdvisorName(serviceDirectory.getAdvisorName(serviceInfo.getName()));
@@ -213,6 +214,9 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem
if (serviceInfo.getServicePackageFolder() == null) {
serviceInfo.setServicePackageFolder(parent.getServicePackageFolder());
}
+ if (serviceInfo.getServiceUpgradesFolder() == null) {
+ serviceInfo.setServiceUpgradesFolder(parent.getServiceUpgradesFolder());
+ }
if (serviceInfo.getMetricsFile() == null) {
serviceInfo.setMetricsFile(parent.getMetricsFile());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
index 13d5047..ee9e383 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
@@ -34,15 +34,13 @@ import org.slf4j.LoggerFactory;
import javax.xml.bind.JAXBException;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
-import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
/**
* Encapsulates IO operations on a stack definition stack directory.
@@ -417,10 +415,11 @@ public class StackDirectory extends StackDefinitionDirectory {
serviceDirectories = dirs;
}
+
/**
* Parse all stack upgrade files for the stack.
*
- * @param subDirs stack sub directories
+ * @param subDirs stack sub directories
* @throws AmbariException if unable to parse stack upgrade file
*/
private void parseUpgradePacks(Collection<String> subDirs) throws AmbariException {
@@ -432,28 +431,19 @@ public class StackDirectory extends StackDefinitionDirectory {
upgradesDir = f.getAbsolutePath();
for (File upgradeFile : f.listFiles(XML_FILENAME_FILTER)) {
if (upgradeFile.getName().toLowerCase().startsWith(CONFIG_UPGRADE_XML_FILENAME_PREFIX)) {
- try { // Parse config upgrade pack
- if (configUpgradePack == null) {
- configUpgradePack = unmarshaller.unmarshal(ConfigUpgradePack.class, upgradeFile);
- } else { // If user messed things up with lower/upper case filenames
- throw new AmbariException(String.format("There are multiple files with name like %s" +
- upgradeFile.getAbsolutePath()));
- }
- } catch (JAXBException e) {
- throw new AmbariException("Unable to parse stack upgrade file at location: " +
- upgradeFile.getAbsolutePath(), e);
+ if (configUpgradePack == null) {
+ configUpgradePack = parseConfigUpgradePack(upgradeFile);
}
- } else {
- try {
- String upgradePackName = FilenameUtils.removeExtension(upgradeFile.getName());
- UpgradePack pack = unmarshaller.unmarshal(UpgradePack.class, upgradeFile);
- pack.setName(upgradePackName);
- upgradeMap.put(upgradePackName, pack);
- } catch (JAXBException e) {
- throw new AmbariException("Unable to parse stack upgrade file at location: " +
- upgradeFile.getAbsolutePath(), e);
+ else { // If user messed things up with lower/upper case filenames
+ throw new AmbariException(String.format("There are multiple files with name like %s" + upgradeFile.getAbsolutePath()));
}
}
+ else {
+ String upgradePackName = FilenameUtils.removeExtension(upgradeFile.getName());
+ UpgradePack pack = parseUpgradePack(upgradePackName, upgradeFile);
+ pack.setName(upgradePackName);
+ upgradeMap.put(upgradePackName, pack);
+ }
}
}
}
@@ -462,7 +452,7 @@ public class StackDirectory extends StackDefinitionDirectory {
LOG.info("Stack '{}' doesn't contain an upgrade directory ", getPath());
}
- if (! upgradeMap.isEmpty()) {
+ if (!upgradeMap.isEmpty()) {
upgradePacks = upgradeMap;
}
@@ -471,7 +461,35 @@ public class StackDirectory extends StackDefinitionDirectory {
} else {
LOG.info("Stack '{}' doesn't contain config upgrade pack file", getPath());
}
+ }
+ private UpgradePack parseUpgradePack(final String packName, File upgradeFile) throws AmbariException {
+ UpgradePack pack = null;
+ try {
+ pack = unmarshaller.unmarshal(UpgradePack.class, upgradeFile);
+ pack.setName(packName);
+ }
+ catch (JAXBException e) {
+ if (upgradeFile == null) {
+ throw new AmbariException("Null upgrade pack");
+ }
+ throw new AmbariException("Unable to parse stack upgrade file at location: " + upgradeFile.getAbsolutePath(), e);
+ }
+ return pack;
+ }
+
+ private ConfigUpgradePack parseConfigUpgradePack(File upgradeFile) throws AmbariException {
+ ConfigUpgradePack pack = null;
+ try {
+ pack = unmarshaller.unmarshal(ConfigUpgradePack.class, upgradeFile);
+ }
+ catch (JAXBException e) {
+ if (upgradeFile == null) {
+ throw new AmbariException("Null config upgrade pack");
+ }
+ throw new AmbariException("Unable to parse stack upgrade file at location: " + upgradeFile.getAbsolutePath(), e);
+ }
+ return pack;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 5a18b3f..d819a52 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -18,25 +18,39 @@
package org.apache.ambari.server.stack;
+import java.io.File;
+import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
+import javax.xml.bind.JAXBException;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.stack.StackDefinitionDirectory;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.stack.ConfigUpgradePack;
import org.apache.ambari.server.state.stack.RepositoryXml;
import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
import org.apache.ambari.server.state.stack.StackMetainfoXml;
+import org.apache.ambari.server.state.stack.UpgradePack;
+import org.apache.ambari.server.state.stack.UpgradePack.OrderService;
+import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping;
+import org.apache.ambari.server.state.stack.upgrade.Grouping;
+import org.apache.ambari.server.state.stack.upgrade.ServiceCheckGrouping;
+import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping.ExecuteStage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,6 +127,11 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
protected boolean valid = true;
/**
+ * file unmarshaller
+ */
+ ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller();
+
+ /**
* Logger
*/
private final static Logger LOG = LoggerFactory.getLogger(StackModule.class);
@@ -155,6 +174,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
if (parentVersion != null) {
mergeStackWithParent(parentVersion, allStacks, commonServices);
}
+ processUpgradePacks();
processRepositories();
processPropertyDependencies();
moduleState = ModuleState.RESOLVED;
@@ -644,6 +664,219 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
}
/**
+ * Process upgrade packs associated with the stack.
+ * @throws AmbariException if unable to fully process the upgrade packs
+ */
+ private void processUpgradePacks() throws AmbariException {
+ if (stackInfo.getUpgradePacks() == null) {
+ return;
+ }
+
+ for (UpgradePack pack : stackInfo.getUpgradePacks().values()) {
+ List<UpgradePack> servicePacks = new ArrayList<>();
+ for (ServiceModule module : serviceModules.values()) {
+ File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder();
+ if (upgradesFolder != null) {
+ UpgradePack servicePack = getServiceUpgradePack(pack, upgradesFolder);
+ if (servicePack != null) {
+ servicePacks.add(servicePack);
+ }
+ }
+ }
+ if (servicePacks.size() > 0) {
+ LOG.info("Merging service specific upgrades for pack: " + pack.getName());
+ mergeUpgradePack(pack, servicePacks);
+ }
+ }
+
+ ConfigUpgradePack configPack = stackInfo.getConfigUpgradePack();
+ if (configPack == null) {
+ return;
+ }
+ for (ServiceModule module : serviceModules.values()) {
+ File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder();
+ if (upgradesFolder != null) {
+ mergeConfigUpgradePack(configPack, upgradesFolder);
+ }
+ }
+ }
+
+ /**
+ * Attempts to merge, into the stack config upgrade, all the config upgrades
+ * for any service which specifies its own upgrade.
+ */
+ private void mergeConfigUpgradePack(ConfigUpgradePack pack, File upgradesFolder) throws AmbariException {
+ File stackFolder = new File(upgradesFolder, stackInfo.getName());
+ File versionFolder = new File(stackFolder, stackInfo.getVersion());
+ File serviceConfig = new File(versionFolder, StackDefinitionDirectory.CONFIG_UPGRADE_XML_FILENAME_PREFIX);
+ if (!serviceConfig.exists()) {
+ return;
+ }
+
+ try {
+ ConfigUpgradePack serviceConfigPack = unmarshaller.unmarshal(ConfigUpgradePack.class, serviceConfig);
+ pack.services.addAll(serviceConfigPack.services);
+ }
+ catch (JAXBException e) {
+ throw new AmbariException("Unable to parse service config upgrade file at location: " + serviceConfig.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * Returns the upgrade pack for a service if it exists, otherwise returns null
+ */
+ private UpgradePack getServiceUpgradePack(UpgradePack pack, File upgradesFolder) throws AmbariException {
+ File stackFolder = new File(upgradesFolder, stackInfo.getName());
+ File versionFolder = new File(stackFolder, stackInfo.getVersion());
+ File servicePackFile = new File(versionFolder, pack.getName() + ".xml");
+ LOG.info("Service folder: " + servicePackFile.getAbsolutePath());
+ if (!servicePackFile.exists()) {
+ return null;
+ }
+ return parseServiceUpgradePack(pack, servicePackFile);
+ }
+
+ /**
+ * Attempts to merge, into the stack upgrade, all the upgrades
+ * for any service which specifies its own upgrade.
+ */
+ private void mergeUpgradePack(UpgradePack pack, List<UpgradePack> servicePacks) throws AmbariException {
+ List<Grouping> originalGroups = pack.getAllGroups();
+ Map<String, List<Grouping>> allGroupMap = new HashMap<>();
+ for (Grouping group : originalGroups) {
+ List<Grouping> list = new ArrayList<>();
+ list.add(group);
+ allGroupMap.put(group.name, list);
+ }
+ for (UpgradePack servicePack : servicePacks) {
+ for (Grouping group : servicePack.getAllGroups()) {
+ if (allGroupMap.containsKey(group.name)) {
+ List<Grouping> list = allGroupMap.get(group.name);
+ Grouping first = list.get(0);
+ if (!first.getClass().equals(group.getClass())) {
+ throw new AmbariException("Expected class: " + first.getClass() + " instead of " + group.getClass());
+ }
+ /* If the current group doesn't specify an "after entry" and the first group does
+ then the current group should be added first. The first group in the list should
+ never be ordered relative to any other group. */
+ if (group.addAfterGroupEntry == null && first.addAfterGroupEntry != null) {
+ list.add(0, group);
+ }
+ else {
+ list.add(group);
+ }
+ }
+ else {
+ List<Grouping> list = new ArrayList<>();
+ list.add(group);
+ allGroupMap.put(group.name, list);
+ }
+ }
+ }
+
+ Map<String, Grouping> mergedGroupMap = new HashMap<>();
+ for (String key : allGroupMap.keySet()) {
+ Iterator<Grouping> iterator = allGroupMap.get(key).iterator();
+ Grouping group = iterator.next();
+ if (iterator.hasNext()) {
+ group.merge(iterator);
+ }
+ mergedGroupMap.put(key, group);
+ }
+
+ orderGroups(originalGroups, mergedGroupMap);
+ }
+
+ /**
+ * Orders the upgrade groups. All new groups specified in a service's upgrade file must
+ * specify after which group they should be placed in the upgrade order.
+ */
+ private void orderGroups(List<Grouping> groups, Map<String, Grouping> mergedGroupMap) throws AmbariException {
+ Map<String, List<Grouping>> skippedGroups = new HashMap<>();
+ for (Map.Entry<String, Grouping> entry : mergedGroupMap.entrySet()) {
+ String key = entry.getKey();
+ Grouping group = entry.getValue();
+ if (!groups.contains(group)) {
+ boolean added = addGrouping(groups, group);
+ if (added) {
+ addSkippedGroup(groups, skippedGroups, group);
+ } else {
+ List<Grouping> tmp = null;
+ // store the group until later
+ if (skippedGroups.containsKey(group.addAfterGroup)) {
+ tmp = skippedGroups.get(group.addAfterGroup);
+ } else {
+ tmp = new ArrayList<Grouping>();
+ skippedGroups.put(group.addAfterGroup, tmp);
+ }
+ tmp.add(group);
+ }
+ }
+ }
+ if (!skippedGroups.isEmpty()) {
+ throw new AmbariException("Missing groups: " + skippedGroups.keySet());
+ }
+ }
+
+ /**
+ * Adds the group provided if the group which it should come after has been added.
+ */
+ private boolean addGrouping(List<Grouping> groups, Grouping group) throws AmbariException {
+ if (group.addAfterGroup == null) {
+ throw new AmbariException("Group " + group.name + " needs to specify which group it should come after");
+ }
+ else {
+ // Check the current services, if the "after" service is there then add these
+ for (int index = groups.size() - 1; index >= 0; index--) {
+ String name = groups.get(index).name;
+ if (name.equals(group.addAfterGroup)) {
+ groups.add(index + 1, group);
+ LOG.debug("Added group/after: " + group.name + "/" + group.addAfterGroup);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Adds any groups which have been previously skipped if the group which they should come
+ * after have been added.
+ */
+ private void addSkippedGroup(List<Grouping> groups, Map<String, List<Grouping>> skippedGroups, Grouping groupJustAdded) throws AmbariException {
+ if (skippedGroups.containsKey(groupJustAdded.name)) {
+ List<Grouping> groupsToAdd = skippedGroups.remove(groupJustAdded.name);
+ for (Grouping group : groupsToAdd) {
+ boolean added = addGrouping(groups, group);
+ if (added) {
+ addSkippedGroup(groups, skippedGroups, group);
+ } else {
+ throw new AmbariException("Failed to add group " + group.name);
+ }
+ }
+ }
+ }
+
+ /**
+ * Parses the service specific upgrade file and merges the none order elements
+ * (prerequisite check and processing sections).
+ */
+ private UpgradePack parseServiceUpgradePack(UpgradePack parent, File serviceFile) throws AmbariException {
+ UpgradePack pack = null;
+ try {
+ pack = unmarshaller.unmarshal(UpgradePack.class, serviceFile);
+ }
+ catch (JAXBException e) {
+ throw new AmbariException("Unable to parse service upgrade file at location: " + serviceFile.getAbsolutePath(), e);
+ }
+
+ parent.mergePrerequisiteChecks(pack);
+ parent.mergeProcessing(pack);
+
+ return pack;
+ }
+
+ /**
* Process repositories associated with the stack.
* @throws AmbariException if unable to fully process the stack repositories
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
index 88f6e19..68c1dd6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
@@ -87,16 +87,34 @@ public class StackServiceDirectory extends ServiceDirectory {
int fileCount = files.length;
if (fileCount > 0) {
packageDir = absPackageDir.getPath().substring(stackDir.getParentFile().getParentFile().getPath().length() + 1);
- LOG.debug(String.format("Service package folder for service %s for stack %s has been resolved to %s",
- serviceDir.getName(), stackId, packageDir));
+ LOG.debug("Service package folder for service %s for stack %s has been resolved to %s",
+ serviceDir.getName(), stackId, packageDir);
}
else {
- LOG.debug(String.format("Service package folder %s for service %s for stack %s is empty.",
- absPackageDir, serviceDir.getName(), stackId));
+ LOG.debug("Service package folder %s for service %s for stack %s is empty.",
+ absPackageDir, serviceDir.getName(), stackId);
}
} else {
- LOG.debug(String.format("Service package folder %s for service %s for stack %s does not exist.",
- absPackageDir, serviceDir.getName(), stackId));
+ LOG.debug("Service package folder %s for service %s for stack %s does not exist.",
+ absPackageDir, serviceDir.getName(), stackId);
+ }
+
+ File absUpgradesDir = new File(getAbsolutePath() + File.separator + UPGRADES_FOLDER_NAME);
+ if (absUpgradesDir.isDirectory()) {
+ String[] files = absUpgradesDir.list();
+ int fileCount = files.length;
+ if (fileCount > 0) {
+ upgradesDir = absUpgradesDir;
+ LOG.debug("Service upgrades folder for service %s for stack %s has been resolved to %s",
+ serviceDir.getName(), stackId, packageDir);
+ }
+ else {
+ LOG.debug("Service upgrades folder %s for service %s for stack %s is empty.",
+ absUpgradesDir, serviceDir.getName(), stackId);
+ }
+ } else {
+ LOG.debug("Service upgrades folder %s for service %s for stack %s does not exist.",
+ absUpgradesDir, serviceDir.getName(), stackId);
}
parseMetaInfoFile();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
index 43cefb9..76840ea 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
@@ -261,6 +261,12 @@ public class ServiceInfo implements Validable{
@XmlTransient
private String servicePackageFolder;
+ /**
+ * Stores the path to the upgrades folder which contains the upgrade xmls for the given service.
+ */
+ @XmlTransient
+ private File serviceUpgradesFolder;
+
public boolean isDeleted() {
return isDeleted;
}
@@ -583,6 +589,14 @@ public String getVersion() {
this.servicePackageFolder = servicePackageFolder;
}
+ public File getServiceUpgradesFolder() {
+ return serviceUpgradesFolder;
+ }
+
+ public void setServiceUpgradesFolder(File serviceUpgradesFolder) {
+ this.serviceUpgradesFolder = serviceUpgradesFolder;
+ }
+
/**
* Exposes (and initializes on first use) map of os-specific details.
* @return map of OS specific details keyed by family
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
index b860731..486d432 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
@@ -154,6 +154,74 @@ public class UpgradePack {
}
/**
+ * Merges the prerequisite checks section of the upgrade xml with
+ * the prerequisite checks from a service's upgrade xml.
+ * These are added to the end of the current list of checks.
+ *
+ * @param pack
+ * the service's upgrade pack
+ */
+ public void mergePrerequisiteChecks(UpgradePack pack) {
+ PrerequisiteChecks newPrereqChecks = pack.prerequisiteChecks;
+ if (prerequisiteChecks == null) {
+ prerequisiteChecks = newPrereqChecks;
+ return;
+ }
+
+ if (newPrereqChecks == null) {
+ return;
+ }
+
+ if (prerequisiteChecks.checks == null) {
+ prerequisiteChecks.checks = new ArrayList<String>();
+ }
+ if (newPrereqChecks.checks != null) {
+ prerequisiteChecks.checks.addAll(newPrereqChecks.checks);
+ }
+
+ if (newPrereqChecks.configuration == null) {
+ return;
+ }
+
+ if (prerequisiteChecks.configuration == null) {
+ prerequisiteChecks.configuration = newPrereqChecks.configuration;
+ return;
+ }
+ if (prerequisiteChecks.configuration.globalProperties == null) {
+ prerequisiteChecks.configuration.globalProperties = new ArrayList<PrerequisiteProperty>();
+ }
+ if (prerequisiteChecks.configuration.prerequisiteCheckProperties == null) {
+ prerequisiteChecks.configuration.prerequisiteCheckProperties = new ArrayList<PrerequisiteCheckProperties>();
+ }
+ if (newPrereqChecks.configuration.globalProperties != null) {
+ prerequisiteChecks.configuration.globalProperties.addAll(newPrereqChecks.configuration.globalProperties);
+ }
+ if (newPrereqChecks.configuration.prerequisiteCheckProperties != null) {
+ prerequisiteChecks.configuration.prerequisiteCheckProperties.addAll(newPrereqChecks.configuration.prerequisiteCheckProperties);
+ }
+ }
+
+/**
+ * Merges the processing section of the upgrade xml with
+ * the processing section from a service's upgrade xml.
+ * These are added to the end of the current list of services.
+ *
+ * @param pack
+ * the service's upgrade pack
+ */
+ public void mergeProcessing(UpgradePack pack) {
+ List<ProcessingService> list = pack.processing;
+ if (list == null) {
+ return;
+ }
+ if (processing == null) {
+ processing = list;
+ return;
+ }
+ processing.addAll(list);
+ }
+
+ /**
* Gets a list of stacks which are between the current stack version and the
* target stack version inclusive. For example, if upgrading from HDP-2.2 to
* HDP-2.4, this should include HDP-2.3 and HDP-2.4.
@@ -195,6 +263,10 @@ public class UpgradePack {
return skipServiceCheckFailures;
}
+ public List<Grouping> getAllGroups() {
+ return groups;
+ }
+
/**
* Gets the groups defined for the upgrade pack. If a direction is defined for
* a group, it must match the supplied direction to be returned
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
index 3325469..80bb26c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
@@ -19,7 +19,9 @@ package org.apache.ambari.server.state.stack.upgrade;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -33,6 +35,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.stack.HostsType;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Host;
@@ -309,4 +312,71 @@ public class ClusterGrouping extends Grouping {
mt.structuredOut = obj.toString();
}
+
+ /**
+ * Attempts to merge the given cluster groupings. This merges the execute stages
+ * in an order specific manner.
+ */
+ public void merge(Iterator<Grouping> iterator) throws AmbariException {
+ if (executionStages == null) {
+ executionStages = new ArrayList<ExecuteStage>();
+ }
+ Map<String, List<ExecuteStage>> skippedStages = new HashMap<>();
+ while (iterator.hasNext()) {
+ Grouping next = iterator.next();
+ if (!(next instanceof ClusterGrouping)) {
+ throw new AmbariException("Invalid group type " + next.getClass().getSimpleName() + " expected cluster group");
+ }
+ ClusterGrouping clusterGroup = (ClusterGrouping) next;
+
+ boolean added = addGroupingStages(clusterGroup.executionStages, clusterGroup.addAfterGroupEntry);
+ if (added) {
+ addSkippedStages(skippedStages, clusterGroup.executionStages);
+ }
+ else {
+ // store these services until later
+ if (skippedStages.containsKey(next.addAfterGroupEntry)) {
+ List<ExecuteStage> tmp = skippedStages.get(clusterGroup.addAfterGroupEntry);
+ tmp.addAll(clusterGroup.executionStages);
+ }
+ else {
+ skippedStages.put(clusterGroup.addAfterGroupEntry, clusterGroup.executionStages);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds the given stages if the stage they are supposed to come after has been added.
+ */
+ private boolean addGroupingStages(List<ExecuteStage> stagesToAdd, String after) {
+ if (after == null) {
+ executionStages.addAll(stagesToAdd);
+ return true;
+ }
+ else {
+ // Check the current stages, if the "after" stage is there then add these
+ for (int index = executionStages.size() - 1; index >= 0; index--) {
+ ExecuteStage stage = executionStages.get(index);
+ if ((stage.service != null && stage.service.equals(after)) || stage.title.equals(after)) {
+ executionStages.addAll(index + 1, stagesToAdd);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Adds the skipped stages if the stage they are supposed to come after has been added.
+ */
+ private void addSkippedStages(Map<String, List<ExecuteStage>> skippedStages, List<ExecuteStage> stagesJustAdded) {
+ for (ExecuteStage stage : stagesJustAdded) {
+ if (skippedStages.containsKey(stage.service)) {
+ List<ExecuteStage> stagesToAdd = skippedStages.remove(stage.service);
+ addGroupingStages(stagesToAdd, stage.service);
+ addSkippedStages(skippedStages, stagesToAdd);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index 67d7fdb..be1f469 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -19,7 +19,9 @@ package org.apache.ambari.server.state.stack.upgrade;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -28,9 +30,11 @@ import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlSeeAlso;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.stack.HostsType;
import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.stack.UpgradePack;
+import org.apache.ambari.server.state.stack.UpgradePack.OrderService;
import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
import org.apache.ambari.server.utils.SetUtils;
import org.apache.commons.lang.StringUtils;
@@ -47,6 +51,12 @@ public class Grouping {
@XmlAttribute(name="title")
public String title;
+ @XmlElement(name="add-after-group")
+ public String addAfterGroup;
+
+ @XmlElement(name="add-after-group-entry")
+ public String addAfterGroupEntry;
+
@XmlElement(name="skippable", defaultValue="false")
public boolean skippable = false;
@@ -310,4 +320,63 @@ public class Grouping {
tasks.add(initial);
}
}
+
+ /**
+ * Merge the services of all the child groups, with the current services.
+ * Keeping the order specified by the group.
+ */
+ public void merge(Iterator<Grouping> iterator) throws AmbariException {
+ Map<String, List<OrderService>> skippedServices = new HashMap<>();
+ while (iterator.hasNext()) {
+ Grouping group = iterator.next();
+
+ boolean added = addGroupingServices(group.services, group.addAfterGroupEntry);
+ if (added) {
+ addSkippedServices(skippedServices, group.services);
+ } else {
+ // store these services until later
+ if (skippedServices.containsKey(group.addAfterGroupEntry)) {
+ List<OrderService> tmp = skippedServices.get(group.addAfterGroupEntry);
+ tmp.addAll(group.services);
+ } else {
+ skippedServices.put(group.addAfterGroupEntry, group.services);
+ }
+ }
+ }
+ }
+
+ /**
+ * Merge the services to add after a particular service name
+ */
+ private boolean addGroupingServices(List<OrderService> servicesToAdd, String after) {
+ if (after == null) {
+ services.addAll(servicesToAdd);
+ return true;
+ }
+ else {
+ // Check the current services, if the "after" service is there then add these
+ for (int index = services.size() - 1; index >= 0; index--) {
+ OrderService service = services.get(index);
+ if (service.serviceName.equals(after)) {
+ services.addAll(index + 1, servicesToAdd);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Adds services which were previously skipped, if the service they are supposed
+ * to come after has been added.
+ */
+ private void addSkippedServices(Map<String, List<OrderService>> skippedServices, List<OrderService> servicesJustAdded) {
+ for (OrderService service : servicesJustAdded) {
+ if (skippedServices.containsKey(service.serviceName)) {
+ List<OrderService> servicesToAdd = skippedServices.remove(service.serviceName);
+ addGroupingServices(servicesToAdd, service.serviceName);
+ addSkippedServices(skippedServices, servicesToAdd);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java
index 5cda422..6a085f3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ServiceCheckGrouping.java
@@ -19,7 +19,9 @@ package org.apache.ambari.server.state.stack.upgrade;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -81,6 +83,13 @@ public class ServiceCheckGrouping extends Grouping {
}
/**
+ * @return the set of service names that should be excluded
+ */
+ public Set<String> getExcluded() {
+ return excludeServices;
+ }
+
+ /**
* Used to build stages for service check groupings.
*/
public class ServiceCheckBuilder extends StageWrapperBuilder {
@@ -187,4 +196,73 @@ public class ServiceCheckGrouping extends Grouping {
return false;
}
}
+
+ /**
+ * Attempts to merge all the service check groupings. This merges the excluded list and
+ * the priorities. The priorities are merged in an order specific manner.
+ */
+ public void merge(Iterator<Grouping> iterator) throws AmbariException {
+ List<String> priorities = new ArrayList<>();
+ priorities.addAll(getPriorities());
+ Map<String, Set<String>> skippedPriorities = new HashMap<>();
+ while (iterator.hasNext()) {
+ Grouping next = iterator.next();
+ if (!(next instanceof ServiceCheckGrouping)) {
+ throw new AmbariException("Invalid group type " + next.getClass().getSimpleName() + " expected service check group");
+ }
+ ServiceCheckGrouping checkGroup = (ServiceCheckGrouping) next;
+ getExcluded().addAll(checkGroup.getExcluded());
+
+ boolean added = addPriorities(priorities, checkGroup.getPriorities(), checkGroup.addAfterGroupEntry);
+ if (added) {
+ addSkippedPriorities(priorities, skippedPriorities, checkGroup.getPriorities());
+ }
+ else {
+ // store these services until later
+ if (skippedPriorities.containsKey(checkGroup.addAfterGroupEntry)) {
+ Set<String> tmp = skippedPriorities.get(checkGroup.addAfterGroupEntry);
+ tmp.addAll(checkGroup.getPriorities());
+ }
+ else {
+ skippedPriorities.put(checkGroup.addAfterGroupEntry, checkGroup.getPriorities());
+ }
+ }
+ }
+ getPriorities().clear();
+ getPriorities().addAll(priorities);
+ }
+
+ /**
+ * Add the given child priorities if the service they are supposed to come after have been added.
+ */
+ private boolean addPriorities(List<String> priorities, Set<String> childPriorities, String after) {
+ if (after == null) {
+ priorities.addAll(childPriorities);
+ return true;
+ }
+ else {
+ // Check the current priorities, if the "after" priority is there then add these
+ for (int index = priorities.size() - 1; index >= 0; index--) {
+ String priority = priorities.get(index);
+ if (after.equals(priority)) {
+ priorities.addAll(index + 1, childPriorities);
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Add the skipped priorities if the services they are supposed to come after have been added
+ */
+ private void addSkippedPriorities(List<String> priorities, Map<String, Set<String>> skippedPriorites, Set<String> prioritiesJustAdded) {
+ for (String priority : prioritiesJustAdded) {
+ if (skippedPriorites.containsKey(priority)) {
+ Set<String> prioritiesToAdd = skippedPriorites.remove(priority);
+ addPriorities(priorities, prioritiesToAdd, priority);
+ addSkippedPriorities(priorities, skippedPriorites, prioritiesToAdd);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index 614487e..712241b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -157,7 +157,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -197,7 +197,7 @@
</batch>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -324,7 +324,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 9fb2bba..525a356 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -154,7 +154,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -193,7 +193,7 @@
</batch>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -311,7 +311,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 1e040e6..9ee3e88 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -160,7 +160,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -199,7 +199,7 @@
</batch>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -317,7 +317,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
index a9ed58d..58aa95f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
@@ -155,7 +155,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -195,7 +195,7 @@
</batch>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -322,7 +322,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 6e27da6..cdb8634 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -155,7 +155,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -194,7 +194,7 @@
</batch>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -312,7 +312,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
index 23b4ca7..d90d76d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
@@ -156,7 +156,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -196,7 +196,7 @@
</batch>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
@@ -332,7 +332,7 @@
</service>
</group>
- <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
+ <group name="SERVICE_CHECK_3" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
<priority>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java
index dda1e7a..850c716 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java
@@ -117,4 +117,35 @@ public class StackManagerMiscTest {
verify(metaInfoDao, stackDao, actionMetadata, osFamily);
}
+
+ /**
+ * This test ensures that service upgrade xml that creates circular dependencies
+ * will throw an exception.
+ */
+ @Test
+ public void testCircularDependencyForServiceUpgrade() throws Exception {
+ MetainfoDAO metaInfoDao = createNiceMock(MetainfoDAO.class);
+ StackDAO stackDao = createNiceMock(StackDAO.class);
+ ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class);
+ OsFamily osFamily = createNiceMock(OsFamily.class);
+ StackEntity stackEntity = createNiceMock(StackEntity.class);
+
+ expect(
+ stackDao.find(EasyMock.anyObject(String.class),
+ EasyMock.anyObject(String.class))).andReturn(stackEntity).atLeastOnce();
+
+ replay(actionMetadata, stackDao, metaInfoDao, osFamily);
+
+ try {
+ String upgradeCycle = ClassLoader.getSystemClassLoader().getResource("stacks_with_upgrade_cycle").getPath();
+
+ StackManager stackManager = new StackManager(new File(upgradeCycle),
+ null, osFamily, metaInfoDao, actionMetadata, stackDao);
+
+ fail("Expected exception due to cyclic service upgrade xml");
+ } catch (AmbariException e) {
+ // expected
+ assertEquals("Missing groups: [BAR, FOO]", e.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
index 15be8b4..c6701b6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
@@ -441,6 +441,72 @@ public class UpgradePackTest {
assertNull(clusterGroup.parallelScheduler);
}
+
+ /**
+ * Tests that the service level XML merges correctly for 2.0.5/HDFS/HDP/2.2.0.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testServiceLevelUpgradePackMerge() throws Exception {
+ Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0");
+ assertTrue(upgrades.containsKey("upgrade_test_15388"));
+
+ UpgradePack upgradePack = upgrades.get("upgrade_test_15388");
+
+ List<String> checks = upgradePack.getPrerequisiteChecks();
+ assertEquals(11, checks.size());
+ assertTrue(checks.contains("org.apache.ambari.server.checks.FooCheck"));
+
+ List<Grouping> groups = upgradePack.getGroups(Direction.UPGRADE);
+ assertEquals(8, groups.size());
+ Grouping group = groups.get(0);
+ assertEquals(ClusterGrouping.class, group.getClass());
+ ClusterGrouping cluster_group = (ClusterGrouping) group;
+ assertEquals("Pre {{direction.text.proper}}", cluster_group.title);
+
+ List<ExecuteStage> stages = cluster_group.executionStages;
+ assertEquals(5, stages.size());
+ ExecuteStage stage = stages.get(3);
+ assertEquals("Backup FOO", stage.title);
+
+ group = groups.get(2);
+ assertEquals("Core Masters", group.title);
+ List<UpgradePack.OrderService> services = group.services;
+ assertEquals(3, services.size());
+ UpgradePack.OrderService service = services.get(2);
+ assertEquals("HBASE", service.serviceName);
+
+ group = groups.get(3);
+ assertEquals("Core Slaves", group.title);
+ services = group.services;
+ assertEquals(3, services.size());
+ service = services.get(1);
+ assertEquals("HBASE", service.serviceName);
+
+ group = groups.get(4);
+ assertEquals(ServiceCheckGrouping.class, group.getClass());
+ ServiceCheckGrouping scGroup = (ServiceCheckGrouping) group;
+ Set<String> priorityServices = scGroup.getPriorities();
+ assertEquals(4, priorityServices.size());
+ Iterator serviceIterator = priorityServices.iterator();
+ assertEquals("ZOOKEEPER", serviceIterator.next());
+ assertEquals("HBASE", serviceIterator.next());
+
+ group = groups.get(5);
+ assertEquals("Hive", group.title);
+
+ group = groups.get(6);
+ assertEquals("Foo", group.title);
+ services = group.services;
+ assertEquals(2, services.size());
+ service = services.get(1);
+ assertEquals("FOO2", service.serviceName);
+
+ Map<String, Map<String, ProcessingComponent>> tasks = upgradePack.getTasks();
+ assertTrue(tasks.containsKey("HBASE"));
+ }
+
private int indexOf(Map<String, ?> map, String keyToFind) {
int result = -1;
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml
new file mode 100644
index 0000000..fd798da
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <target>2.4.*</target>
+ <target-stack>HDP-2.4.0</target-stack>
+ <type>ROLLING</type>
+ <prerequisite-checks>
+ <check>org.apache.ambari.server.checks.FooCheck</check>
+ </prerequisite-checks>
+ <order>
+ <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}">
+ <add-after-group-entry>HDFS</add-after-group-entry>
+ <execute-stage service="FOO" component="BAR" title="Backup FOO">
+ <task xsi:type="manual">
+ <message>Back FOO up.</message>
+ </task>
+ </execute-stage>
+ </group>
+
+ <group name="SERVICE_CHECK1" title="All Service Checks" xsi:type="service-check">
+ <add-after-group-entry>ZOOKEEPER</add-after-group-entry>
+ <priority>
+ <service>HBASE</service>
+ </priority>
+ </group>
+
+ <group name="FOO" title="Foo">
+ <add-after-group>HIVE</add-after-group>
+ <add-after-group-entry>FOO</add-after-group-entry>
+ <skippable>true</skippable>
+ <allow-retry>false</allow-retry>
+ <service name="FOO2">
+ <component>BAR2</component>
+ </service>
+ </group>
+
+ <group name="FOO" title="Foo">
+ <add-after-group>HIVE</add-after-group>
+ <skippable>true</skippable>
+ <allow-retry>false</allow-retry>
+ <service name="FOO">
+ <component>BAR</component>
+ </service>
+ </group>
+
+ <group name="CORE_MASTER" title="Core Masters">
+ <add-after-group-entry>YARN</add-after-group-entry>
+ <service name="HBASE">
+ <component>HBASE_MASTER</component>
+ </service>
+ </group>
+
+ <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
+ <add-after-group-entry>HDFS</add-after-group-entry>
+ <service name="HBASE">
+ <component>REGIONSERVER</component>
+ </service>
+ </group>
+ </order>
+
+ <processing>
+ <service name="HBASE">
+ <component name="HBASE_MASTER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
+ </pre-upgrade>
+ <pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="HBASE_REGIONSERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+ </processing>
+</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml
new file mode 100644
index 0000000..4e8e2ea
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml
@@ -0,0 +1,232 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <target>2.4.*</target>
+ <target-stack>HDP-2.4.0</target-stack>
+ <type>ROLLING</type>
+ <prerequisite-checks>
+ <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check>
+ <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check>
+ <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check>
+ <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check>
+ <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check>
+ <check>org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck</check>
+ <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check>
+ <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check>
+ <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check>
+ <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check>
+ </prerequisite-checks>
+ <order>
+ <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}">
+ <execute-stage title="Confirm 1">
+ <task xsi:type="manual">
+ <message>Foo</message>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Pre Upgrade HIVE">
+ <task xsi:type="manual">
+ <message>Back stuff up.</message>
+ </task>
+ </execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS">
+ <task xsi:type="execute">
+ <command>ls</command>
+ </task>
+ </execute-stage>
+ <execute-stage title="Confirm 2">
+ <task xsi:type="manual">
+ <message>Goo</message>
+ </task>
+ </execute-stage>
+ </group>
+
+ <group name="ZOOKEEPER" title="Zookeeper">
+ <skippable>true</skippable>
+ <allow-retry>false</allow-retry>
+ <service name="ZOOKEEPER">
+ <component>ZOOKEEPER_SERVER</component>
+ <component>ZOOKEEPER_CLIENT</component>
+ </service>
+ </group>
+
+ <group name="CORE_MASTER" title="Core Masters">
+ <service name="HDFS">
+ <component>JOURNALNODE</component>
+ <component>NAMENODE</component>
+ </service>
+ <service name="YARN">
+ <component>RESOURCEMANAGER</component>
+ </service>
+ </group>
+
+ <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
+ <skippable>true</skippable> <!-- set skippable for test -->
+ <allow-retry>false</allow-retry> <!-- set no retry for test -->
+ <service name="HDFS">
+ <component>DATANODE</component>
+ </service>
+ <service name="YARN">
+ <component>NODEMANAGER</component>
+ </service>
+
+ <batch>
+ <percent>20</percent>
+ <message>Please run additional tests on {{components}}</message>
+ </batch>
+ </group>
+
+ <group name="SERVICE_CHECK1" title="All Service Checks" xsi:type="service-check">
+ <skippable>true</skippable>
+ <direction>UPGRADE</direction>
+ <priority>
+ <service>ZOOKEEPER</service>
+ <service>HDFS</service>
+ <service>YARN</service>
+ </priority>
+ <exclude>
+ <service>AMBARI_METRICS</service>
+ <service>LOGSEARCH</service>
+ </exclude>
+ </group>
+
+ <group name="HIVE" title="Hive">
+ <skippable>true</skippable>
+ <service name="HIVE">
+ <component>HIVE_METASTORE</component>
+ <component>HIVE_SERVER</component>
+ <component>WEBHCAT_SERVER</component>
+ </service>
+ </group>
+
+ <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
+ <execute-stage title="Confirm Finalize">
+ <task xsi:type="manual">
+ <message>Please confirm you are ready to finalize</message>
+ </task>
+ </execute-stage>
+ <execute-stage title="Update remaining HDP stack to {{version}}">
+ <task xsi:type="execute">
+ <script>scripts/ru_set_all.py</script>
+ <function>actionexecute</function>
+ </task>
+ </execute-stage>
+ <execute-stage title="Save Cluster State" service="" component="">
+ <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
+ </task>
+ </execute-stage>
+ </group>
+
+ </order>
+
+
+ <processing>
+ <service name="ZOOKEEPER">
+ <component name="ZOOKEEPER_SERVER">
+ <pre-upgrade>
+ <task xsi:type="manual">
+ <summary>SUMMARY OF PREPARE</summary>
+ <message>This is a manual task with a placeholder of {{foo/bar}}</message>
+ </task>
+ </pre-upgrade>
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ <post-upgrade>
+ <task xsi:type="configure" />
+ </post-upgrade>
+ </component>
+ </service>
+
+ <service name="HDFS">
+ <component name="NAMENODE">
+ <pre-upgrade>
+ <task xsi:type="execute" hosts="master">
+ <command>su - {hdfs-user} -c 'dosomething'</command>
+ </task>
+ <task xsi:type="configure">
+ <type>hdfs-site</type>
+ <set key="myproperty" value="mynewvalue"/>
+ </task>
+ <task xsi:type="manual">
+ <message>{{direction.verb.proper}} your database</message>
+ </task>
+ </pre-upgrade>
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ <post-upgrade>
+ <task xsi:type="execute">
+ <command>ls</command>
+ </task>
+ </post-upgrade>
+ </component>
+ <component name="DATANODE">
+ <pre-downgrade />
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ <post-downgrade>
+ <task xsi:type="manual">
+ <message>Manual Downgrade</message>
+ </task>
+ </post-downgrade>
+ </component>
+ </service>
+
+ <service name="YARN">
+ <component name="RESOURCEMANAGER">
+ <pre-upgrade>
+ <task xsi:type="execute">
+ <command>ls</command>
+ </task>
+ </pre-upgrade>
+ </component>
+ <component name="NODEMANAGER">
+ <pre-upgrade>
+ <task xsi:type="execute">
+ <command>ls</command>
+ </task>
+ </pre-upgrade>
+ </component>
+ </service>
+
+ <service name="HIVE">
+ <component name="HIVE_SERVER">
+ <pre-upgrade>
+ <task xsi:type="manual">
+ <summary>HiveServer Port Availability</summary>
+ <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message>
+ </task>
+
+ <task xsi:type="configure">
+ <condition type="hive-site" key="hive.server2.transport.mode" value="binary">
+ <type>hive-site</type>
+ <key>hive.server2.thrift.port</key>
+ <value>10010</value>
+ </condition>
+ <condition type="hive-site" key="hive.server2.transport.mode" value="http">
+ <type>hive-site</type>
+ <key>hive.server2.http.port</key>
+ <value>10011</value>
+ </condition>
+ </task>
+ </pre-upgrade>
+ </component>
+ </service>
+ </processing>
+</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml
new file mode 100644
index 0000000..5ffea3a
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/metainfo.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <versions>
+ <active>true</active>
+ </versions>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json
new file mode 100644
index 0000000..f80fc0a
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/hdp.json
@@ -0,0 +1,15 @@
+{
+ "HDP-2.2.0": {
+ "latest": {
+ "centos6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.0.0-123",
+ "redhat6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.2.0-123",
+ "oraclelinux6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.0.0-123",
+ "suse11": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11/2.x/BUILDS/2.2.0.0-123/hdp.repo"
+ },
+ "manifests": {
+ "2.2.1.0": {
+ "centos6": "./version-2.2.0.4-123.xml"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml
new file mode 100644
index 0000000..207d258
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/repoinfo.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <latest>./hdp.json</latest>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0</baseurl>
+ <repoid>HDP-2.2.0</repoid>
+ <reponame>HDP</reponame>
+ </repo>
+ </os>
+ <os family="suse11">
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.2.0.0</baseurl>
+ <repoid>HDP-2.2.0</repoid>
+ <reponame>HDP</reponame>
+ </repo>
+ </os>
+</reposinfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml
new file mode 100644
index 0000000..0b027ee
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/repos/version-2.2.0.4-123.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="version_definition.xsd">
+ <release>
+ <type>STANDARD</type>
+ <stack-id>HDP-2.2.0</stack-id>
+ <version>2.2.0.4</version>
+ <build>123</build>
+ <release-notes>http://example.com</release-notes>
+ <display>HDP-2.2.0.4-1234</display>
+ <compatible-with>2.2.0.0</compatible-with>
+ </release>
+ <manifest>
+ <service id="HDFS-271" name="HDFS" version="2.7.1.2.4"/>
+ <service id="HBASE-132" name="HBASE" version="1.3.2.4.3"/>
+ </manifest>
+ <available-services>
+ <service idref="HDFS-271"/>
+ </available-services>
+ <repository-info>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://baseurl1</baseurl>
+ <repoid>HDP-2.4</repoid>
+ <reponame>HDP</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://baseurl2</baseurl>
+ <repoid>HDP-UTILS-1.1.0.20</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+ </repository-info>
+</repository-version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b42749/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json
new file mode 100644
index 0000000..ebac7cf
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/role_command_order.json
@@ -0,0 +1,80 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"],
+ "OOZIE_SERVER-START": ["JOBTRACKER-START", "TASKTRACKER-START"],
+ "WEBHCAT_SERVER-START": ["TASKTRACKER-START", "HIVE_SERVER-START"],
+ "HIVE_METASTORE-START": ["MYSQL_SERVER-START"],
+ "HIVE_SERVER-START": ["TASKTRACKER-START", "MYSQL_SERVER-START"],
+ "HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"],
+ "FLUME_HANDLER-START": ["OOZIE_SERVER-START"],
+ "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"],
+ "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START"],
+ "WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"],
+ "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
+ "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"],
+ "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"],
+ "PIG_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"],
+ "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"],
+ "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
+ "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
+ "ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP"],
+ "HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"],
+ "TASKTRACKER-UPGRADE": ["JOBTRACKER-UPGRADE"],
+ "MAPREDUCE_CLIENT-UPGRADE": ["TASKTRACKER-UPGRADE", "JOBTRACKER-UPGRADE"],
+ "ZOOKEEPER_SERVER-UPGRADE": ["MAPREDUCE_CLIENT-UPGRADE"],
+ "ZOOKEEPER_CLIENT-UPGRADE": ["ZOOKEEPER_SERVER-UPGRADE"],
+ "HBASE_MASTER-UPGRADE": ["ZOOKEEPER_CLIENT-UPGRADE"],
+ "HBASE_REGIONSERVER-UPGRADE": ["HBASE_MASTER-UPGRADE"],
+ "HBASE_CLIENT-UPGRADE": ["HBASE_REGIONSERVER-UPGRADE"],
+ "HIVE_SERVER-UPGRADE" : ["HBASE_CLIENT-UPGRADE"],
+ "HIVE_METASTORE-UPGRADE" : ["HIVE_SERVER-UPGRADE"],
+ "MYSQL_SERVER-UPGRADE": ["HIVE_METASTORE-UPGRADE"],
+ "HIVE_CLIENT-UPGRADE": ["MYSQL_SERVER-UPGRADE"],
+ "HCAT-UPGRADE": ["HIVE_CLIENT-UPGRADE"],
+ "OOZIE_SERVER-UPGRADE" : ["HCAT-UPGRADE"],
+ "OOZIE_CLIENT-UPGRADE" : ["OOZIE_SERVER-UPGRADE"],
+ "WEBHCAT_SERVER-UPGRADE" : ["OOZIE_CLIENT-UPGRADE"],
+ "PIG-UPGRADE" : ["WEBHCAT_SERVER-UPGRADE"],
+ "SQOOP-UPGRADE" : ["PIG-UPGRADE"],
+ "GANGLIA_SERVER-UPGRADE" : ["SQOOP-UPGRADE"],
+ "GANGLIA_MONITOR-UPGRADE" : ["GANGLIA_SERVER-UPGRADE"]
+ },
+ "_comment" : "GLUSTERFS-specific dependencies",
+ "optional_glusterfs": {
+ "HBASE_MASTER-START": ["PEERSTATUS-START"],
+ "JOBTRACKER-START": ["PEERSTATUS-START"],
+ "TASKTRACKER-START": ["PEERSTATUS-START"],
+ "GLUSTERFS_SERVICE_CHECK-SERVICE_CHECK": ["PEERSTATUS-START"],
+ "JOBTRACKER-UPGRADE": ["GLUSTERFS_CLIENT-UPGRADE"]
+ },
+ "_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster",
+ "optional_no_glusterfs": {
+ "SECONDARY_NAMENODE-START": ["NAMENODE-START"],
+ "RESOURCEMANAGER-START": ["NAMENODE-START", "DATANODE-START"],
+ "NODEMANAGER-START": ["NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START"],
+ "HISTORYSERVER-START": ["NAMENODE-START", "DATANODE-START"],
+ "HBASE_MASTER-START": ["NAMENODE-START", "DATANODE-START"],
+ "JOBTRACKER-START": ["NAMENODE-START", "DATANODE-START"],
+ "TASKTRACKER-START": ["NAMENODE-START", "DATANODE-START"],
+ "HIVE_SERVER-START": ["DATANODE-START"],
+ "WEBHCAT_SERVER-START": ["DATANODE-START"],
+ "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["NAMENODE-START", "DATANODE-START",
+ "SECONDARY_NAMENODE-START"],
+ "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START",
+ "RESOURCEMANAGER-START", "HISTORYSERVER-START", "YARN_SERVICE_CHECK-SERVICE_CHECK"],
+ "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
+ "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"],
+ "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "NAMENODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP",
+ "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"],
+ "DATANODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP",
+ "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"],
+ "SECONDARY_NAMENODE-UPGRADE": ["NAMENODE-UPGRADE"],
+ "DATANODE-UPGRADE": ["SECONDARY_NAMENODE-UPGRADE"],
+ "HDFS_CLIENT-UPGRADE": ["DATANODE-UPGRADE"],
+ "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"]
+ }
+}