You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/12/07 22:00:45 UTC
ambari git commit: AMBARI-18937. Allow generic Service Upgrade Packs
for all targets (ncole)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 070c5439e -> 4453782e2
AMBARI-18937. Allow generic Service Upgrade Packs for all targets (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4453782e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4453782e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4453782e
Branch: refs/heads/branch-2.5
Commit: 4453782e2631677fdc815cadbfcfa82bb555ef19
Parents: 070c543
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Dec 2 16:50:36 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Wed Dec 7 17:00:25 2016 -0500
----------------------------------------------------------------------
.../apache/ambari/server/stack/StackModule.java | 101 +++++++++++++++++--
.../ambari/server/state/stack/UpgradePack.java | 12 ++-
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 4 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 4 +-
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 4 +-
.../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 4 +-
.../src/main/resources/upgrade-pack.xsd | 10 +-
.../server/state/stack/UpgradePackTest.java | 22 ++++
.../GANGLIA/upgrades/HDP/rolling-upgrade.xml | 43 ++++++++
9 files changed, 184 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 37f4167..52d39e0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -44,16 +44,19 @@ import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
import org.apache.ambari.server.state.stack.StackMetainfoXml;
import org.apache.ambari.server.state.stack.UpgradePack;
import org.apache.ambari.server.state.stack.upgrade.Grouping;
+import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
-import com.google.common.collect.ListMultimap;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimaps;
+
/**
* Stack module which provides all functionality related to parsing and fully
* resolving stacks from the stack definition.
@@ -831,6 +834,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
if (configPack == null) {
return;
}
+
for (ServiceModule module : serviceModules.values()) {
File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder();
if (upgradesFolder != null) {
@@ -866,12 +870,17 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
private UpgradePack getServiceUpgradePack(UpgradePack pack, File upgradesFolder) throws AmbariException {
File stackFolder = new File(upgradesFolder, stackInfo.getName());
File versionFolder = new File(stackFolder, stackInfo.getVersion());
+ // !!! relies on the service upgrade pack filename being named the exact same
File servicePackFile = new File(versionFolder, pack.getName() + ".xml");
+
LOG.info("Service folder: " + servicePackFile.getAbsolutePath());
- if (!servicePackFile.exists()) {
- return null;
+ if (servicePackFile.exists()) {
+ return parseServiceUpgradePack(pack, servicePackFile);
+ } else {
+ UpgradePack child = findServiceUpgradePack(pack, stackFolder);
+
+ return null == child ? null : parseServiceUpgradePack(pack, child);
}
- return parseServiceUpgradePack(pack, servicePackFile);
}
/**
@@ -879,6 +888,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
* for any service which specifies its own upgrade.
*/
private void mergeUpgradePack(UpgradePack pack, List<UpgradePack> servicePacks) throws AmbariException {
+
List<Grouping> originalGroups = pack.getAllGroups();
Map<String, List<Grouping>> allGroupMap = new HashMap<>();
for (Grouping group : originalGroups) {
@@ -886,8 +896,21 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
list.add(group);
allGroupMap.put(group.name, list);
}
+
for (UpgradePack servicePack : servicePacks) {
for (Grouping group : servicePack.getAllGroups()) {
+
+ /*
+ !!! special case where the service pack is targeted for any version. When
+ a service UP targets to run after another group, check to make sure that the
+ base UP contains the group.
+ */
+ if (servicePack.isAllTarget() && !allGroupMap.keySet().contains(group.addAfterGroup)) {
+ LOG.warn("Service Upgrade Pack specified after-group of {}, but that is not found in {}",
+ group.addAfterGroup, StringUtils.join(allGroupMap.keySet(), ','));
+ continue;
+ }
+
if (allGroupMap.containsKey(group.name)) {
List<Grouping> list = allGroupMap.get(group.name);
Grouping first = list.get(0);
@@ -903,8 +926,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
else {
list.add(group);
}
- }
- else {
+ } else {
List<Grouping> list = new ArrayList<>();
list.add(group);
allGroupMap.put(group.name, list);
@@ -931,15 +953,17 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
*/
private void orderGroups(List<Grouping> groups, Map<String, Grouping> mergedGroupMap) throws AmbariException {
Map<String, List<Grouping>> skippedGroups = new HashMap<>();
+
for (Map.Entry<String, Grouping> entry : mergedGroupMap.entrySet()) {
- String key = entry.getKey();
Grouping group = entry.getValue();
+
if (!groups.contains(group)) {
boolean added = addGrouping(groups, group);
if (added) {
addSkippedGroup(groups, skippedGroups, group);
} else {
List<Grouping> tmp = null;
+
// store the group until later
if (skippedGroups.containsKey(group.addAfterGroup)) {
tmp = skippedGroups.get(group.addAfterGroup);
@@ -951,6 +975,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
}
}
}
+
if (!skippedGroups.isEmpty()) {
throw new AmbariException("Missing groups: " + skippedGroups.keySet());
}
@@ -996,6 +1021,50 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
}
/**
+ * Finds an upgrade pack that:
+ * <ul>
+ * <li>Is found in the $SERVICENAME/upgrades/$STACKNAME folder</li>
+ * <li>Matches the same {@link UpgradeType#getType()}as the {@code base} upgrade pack</li>
+ * <li>Has the {@link UpgradePack#getTarget()} value equals to "*"</li>
+ * <li>Has the {@link UpgradePack#getTargetStack()} value equals to "*"</li>
+ * </ul>
+ * This method will not attempt to resolve the "most correct" upgrade pack. For this
+ * feature to work, there should be only one upgrade pack per type. If more specificity
+ * is required, then follow the convention of $SERVICENAME/upgrades/$STACKNAME/$STACKVERSION/$BASE_FILE_NAME.xml
+ *
+ * @param base the base upgrade pack for a stack
+ * @param upgradeStackDirectory service directory that contains stack upgrade files.
+ * @return an upgrade pack that matches {@code base}
+ */
+ private UpgradePack findServiceUpgradePack(UpgradePack base, File upgradeStackDirectory) {
+ if (!upgradeStackDirectory.exists() || !upgradeStackDirectory.isDirectory()) {
+ return null;
+ }
+
+ File[] upgradeFiles = upgradeStackDirectory.listFiles(StackDirectory.XML_FILENAME_FILTER);
+ if (0 == upgradeFiles.length) {
+ return null;
+ }
+
+ for (File f : upgradeFiles) {
+ try {
+ UpgradePack upgradePack = unmarshaller.unmarshal(UpgradePack.class, f);
+
+ // !!! if the type is the same and the target is "*", then it's good to merge
+ if (upgradePack.isAllTarget() && upgradePack.getType() == base.getType()) {
+ return upgradePack;
+ }
+
+ } catch (Exception e) {
+ LOG.warn("File {} does not appear to be an upgrade pack and will be skipped ({})",
+ f.getAbsolutePath(), e.getMessage());
+ }
+ }
+
+ return null;
+ }
+
+ /**
* Parses the service specific upgrade file and merges the none order elements
* (prerequisite check and processing sections).
*/
@@ -1008,12 +1077,24 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
throw new AmbariException("Unable to parse service upgrade file at location: " + serviceFile.getAbsolutePath(), e);
}
- parent.mergePrerequisiteChecks(pack);
- parent.mergeProcessing(pack);
+ return parseServiceUpgradePack(parent, pack);
+ }
- return pack;
+ /**
+ * Places prerequisite checks and processing objects onto the parent upgrade pack.
+ *
+ * @param parent the parent upgrade pack
+ * @param child the parsed child upgrade pack
+ * @return the child upgrade pack
+ */
+ private UpgradePack parseServiceUpgradePack(UpgradePack parent, UpgradePack child) {
+ parent.mergePrerequisiteChecks(child);
+ parent.mergeProcessing(child);
+
+ return child;
}
+
/**
* Process repositories associated with the stack.
* @throws AmbariException if unable to fully process the stack repositories
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
index afcab40..76f1897 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
@@ -52,6 +52,8 @@ import org.slf4j.LoggerFactory;
@XmlAccessorType(XmlAccessType.FIELD)
public class UpgradePack {
+ private static final String ALL_VERSIONS = "*";
+
private static Logger LOG = LoggerFactory.getLogger(UpgradePack.class);
/**
@@ -320,7 +322,6 @@ public class UpgradePack {
public boolean canBeApplied(String targetVersion){
// check that upgrade pack can be applied to selected stack
// converting 2.2.*.* -> 2\.2(\.\d+)?(\.\d+)?(-\d+)?
-
String regexPattern = getTarget().replaceAll("\\.", "\\\\."); // . -> \.
regexPattern = regexPattern.replaceAll("\\\\\\.\\*", "(\\\\\\.\\\\d+)?"); // \.* -> (\.\d+)?
regexPattern = regexPattern.concat("(-\\d+)?");
@@ -464,6 +465,15 @@ public class UpgradePack {
}
/**
+ * @return {@code true} if the upgrade targets any version or stack. Both
+ * {@link #target} and {@link #targetStack} must equal "*"
+ */
+ public boolean isAllTarget() {
+ return ALL_VERSIONS.equals(target) && ALL_VERSIONS.equals(targetStack);
+ }
+
+
+ /**
* A service definition that holds a list of components in the 'order' element.
*/
public static class OrderService {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 4d1b5f1..bb70f6a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -435,7 +435,7 @@
</execute-stage>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
<direction>UPGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
@@ -457,7 +457,7 @@
</service>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
<direction>DOWNGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 02cef57..c8baea1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -436,7 +436,7 @@
</execute-stage>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
<direction>UPGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
@@ -458,7 +458,7 @@
</service>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
<direction>DOWNGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 201150d..39b7a4d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -420,7 +420,7 @@
</execute-stage>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
<direction>UPGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
@@ -442,7 +442,7 @@
</service>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
<direction>DOWNGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index cb16953..2fd7a7a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -425,7 +425,7 @@
</execute-stage>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
<direction>UPGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
@@ -447,7 +447,7 @@
</service>
</group>
- <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+ <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
<direction>DOWNGRADE</direction>
<service-check>false</service-check>
<skippable>true</skippable>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/main/resources/upgrade-pack.xsd
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade-pack.xsd b/ambari-server/src/main/resources/upgrade-pack.xsd
index 913473f..aeec803 100644
--- a/ambari-server/src/main/resources/upgrade-pack.xsd
+++ b/ambari-server/src/main/resources/upgrade-pack.xsd
@@ -398,7 +398,15 @@
<xs:element name="type" type="upgrade-kind-type" />
<xs:element name="prerequisite-checks" type="prerequisite-check-type" minOccurs="0" />
<xs:element name="upgrade-path" type="upgrade-path-type" minOccurs="0" />
- <xs:element name="order" type="order-type" />
+ <xs:element name="order" type="order-type">
+ <xs:unique name="unique-by-group">
+ <xs:annotation>
+ <xs:documentation>Ensures that the element "processing" does not have duplicate services</xs:documentation>
+ </xs:annotation>
+ <xs:selector xpath="group" />
+ <xs:field xpath="@name" />
+ </xs:unique>
+ </xs:element>
<xs:element name="processing" type="processing-type">
<xs:unique name="unique-by-service">
<xs:annotation>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
index dde7ffa..8f9f791 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.state.stack;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -570,6 +571,27 @@ public class UpgradePackTest {
Map<String, Map<String, ProcessingComponent>> tasks = upgradePack.getTasks();
assertTrue(tasks.containsKey("HBASE"));
+
+ // !!! generalized upgrade pack shouldn't be in this
+ boolean found = false;
+ for (Grouping grouping : upgradePack.getAllGroups()) {
+ if (grouping.name.equals("GANGLIA_UPGRADE")) {
+ found = true;
+ break;
+ }
+ }
+ assertFalse(found);
+
+ // !!! test merge of a generalized upgrade pack
+ upgradePack = upgrades.get("upgrade_test_conditions");
+ assertNotNull(upgradePack);
+ for (Grouping grouping : upgradePack.getAllGroups()) {
+ if (grouping.name.equals("GANGLIA_UPGRADE")) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue(found);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4453782e/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
new file mode 100644
index 0000000..1d34c59
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-pack.xsd">
+ <target>*</target>
+ <target-stack>*</target-stack>
+ <type>ROLLING</type>
+ <order>
+ <group name="GANGLIA_UPGRADE" title="Ganglia">
+ <add-after-group>ZOOKEEPER_CONFIG_CONDITION_TEST</add-after-group>
+ <skippable>true</skippable>
+ <allow-retry>true</allow-retry>
+ <service name="GANGLIA">
+ <component>HST_SERVER</component>
+ </service>
+ </group>
+ </order>
+
+ <processing>
+ <service name="GANGLIA">
+ <component name="GANGLIA_SERVER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+ </processing>
+</upgrade>