You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2015/02/03 21:43:46 UTC
ambari git commit: AMBARI-9456 - Add a 'Prepare Backups' Stage to
Rolling Upgrade before ZooKeeper (tbeerbower)
Repository: ambari
Updated Branches:
refs/heads/trunk 370d92724 -> a4d964500
AMBARI-9456 - Add a 'Prepare Backups' Stage to Rolling Upgrade before ZooKeeper (tbeerbower)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a4d96450
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a4d96450
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a4d96450
Branch: refs/heads/trunk
Commit: a4d96450030976bdad9f5607cbe0d4ba22a37de1
Parents: 370d927
Author: tbeerbower <tb...@hortonworks.com>
Authored: Tue Feb 3 14:07:10 2015 -0500
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Tue Feb 3 15:37:57 2015 -0500
----------------------------------------------------------------------
.../state/stack/upgrade/ClusterGrouping.java | 133 +++++++++++--------
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 52 +++++---
.../internal/UpgradeResourceProviderTest.java | 6 +-
.../ambari/server/state/UpgradeHelperTest.java | 2 +
.../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 5 +
5 files changed, 120 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a4d96450/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
index 4021d01..91d9960 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
@@ -109,63 +109,24 @@ public class ClusterGrouping extends Grouping {
StageWrapper wrapper = null;
- if (null != execution.service && !execution.service.isEmpty() &&
- null != execution.component && !execution.component.isEmpty()) {
-
- HostsType hosts = ctx.getResolver().getMasterAndHosts(
- execution.service, execution.component);
-
- if (null == hosts) {
- continue;
- }
-
- Set<String> realHosts = new LinkedHashSet<String>(hosts.hosts);
-
- // !!! FIXME other types
- if (task.getType() == Task.Type.EXECUTE) {
- ExecuteTask et = (ExecuteTask) task;
-
- if (null != et.hosts && "master".equals(et.hosts) && null != hosts.master) {
- realHosts = Collections.singleton(hosts.master);
- }
+ switch (task.getType()) {
+ case MANUAL:
+ wrapper = getManualStageWrapper(ctx, execution);
+ break;
+ case SERVER_ACTION:
wrapper = new StageWrapper(
- StageWrapper.Type.RU_TASKS,
+ StageWrapper.Type.SERVER_SIDE_ACTION,
execution.title,
- new TaskWrapper(execution.service, execution.component, realHosts, task));
- }
- } else {
- switch (task.getType()) {
- case MANUAL:
- if (null != execution.id && execution.id.equals("unhealthy-hosts")) {
-
- // !!! this specific task is used ONLY when there are unhealthy
- if (!ctx.getUnhealthy().isEmpty()) {
- ManualTask mt = (ManualTask) task;
-
- fillHostDetails(mt, ctx.getUnhealthy());
-
- wrapper = new StageWrapper(
- StageWrapper.Type.SERVER_SIDE_ACTION,
- execution.title,
- new TaskWrapper(null, null, Collections.<String>emptySet(), task));
- }
- } else {
- wrapper = new StageWrapper(
- StageWrapper.Type.SERVER_SIDE_ACTION,
- execution.title,
- new TaskWrapper(null, null, Collections.<String>emptySet(), task));
- }
- break;
- case SERVER_ACTION:
- wrapper = new StageWrapper(
- StageWrapper.Type.SERVER_SIDE_ACTION,
- execution.title,
- new TaskWrapper(null, null, Collections.<String>emptySet(), task));
- break;
- default:
- break;
- }
+ new TaskWrapper(null, null, Collections.<String>emptySet(), task));
+ break;
+
+ case EXECUTE:
+ wrapper = getExecuteStageWrapper(ctx, execution);
+ break;
+
+ default:
+ break;
}
if (null != wrapper) {
@@ -178,6 +139,70 @@ public class ClusterGrouping extends Grouping {
}
}
+ private StageWrapper getManualStageWrapper(UpgradeContext ctx, ExecuteStage execution) {
+
+ String service = execution.service;
+ String component = execution.component;
+ String id = execution.id;
+ Task task = execution.task;
+
+ if (null != id && id.equals("unhealthy-hosts")) {
+
+ // !!! this specific task is used ONLY when there are unhealthy
+ if (ctx.getUnhealthy().isEmpty()) {
+ return null;
+ }
+ ManualTask mt = (ManualTask) task;
+
+ fillHostDetails(mt, ctx.getUnhealthy());
+ }
+
+ Set<String> realHosts = Collections.emptySet();
+
+ if (null != service && !service.isEmpty() &&
+ null != component && !component.isEmpty()) {
+
+ HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component);
+
+ if (null != hosts) {
+ realHosts = new LinkedHashSet<String>(hosts.hosts);
+ }
+ }
+
+ return new StageWrapper(
+ StageWrapper.Type.SERVER_SIDE_ACTION,
+ execution.title,
+ new TaskWrapper(service, component, realHosts, task));
+ }
+
+ private StageWrapper getExecuteStageWrapper(UpgradeContext ctx, ExecuteStage execution) {
+ String service = execution.service;
+ String component = execution.component;
+ Task task = execution.task;
+
+ if (null != service && !service.isEmpty() &&
+ null != component && !component.isEmpty()) {
+
+ HostsType hosts = ctx.getResolver().getMasterAndHosts(service, component);
+
+ if (hosts != null) {
+ Set<String> realHosts = new LinkedHashSet<String>(hosts.hosts);
+
+ ExecuteTask et = (ExecuteTask) task;
+
+ if (null != et.hosts && "master".equals(et.hosts) && null != hosts.master) {
+ realHosts = Collections.singleton(hosts.master);
+ }
+
+ return new StageWrapper(
+ StageWrapper.Type.RU_TASKS,
+ execution.title,
+ new TaskWrapper(service, component, realHosts, task));
+ }
+ }
+ return null;
+ }
+
private void fillHostDetails(ManualTask mt, Map<String, List<String>> unhealthy) {
JsonArray arr = new JsonArray();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a4d96450/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index b2cc54e..a7ea084 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -21,6 +21,37 @@
<target>2.2.*.*</target>
<order>
+
+ <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups">
+
+ <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS">
+ <task xsi:type="execute" hosts="master">
+ <script>scripts/namenode.py</script>
+ <function>prepare_rolling_upgrade</function>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Pre Upgrade HBASE">
+ <task xsi:type="execute" hosts="master">
+ <script>scripts/hbase_upgrade.py</script>
+ <function>snapshot</function>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="HIVE" component="HIVE_METASTORE" title="Pre Upgrade HIVE">
+ <task xsi:type="manual">
+ <message>Backup the Hive Metastore database located on the following host(s): {{hosts.all}}</message>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Pre Upgrade OOZIE">
+ <task xsi:type="manual">
+ <message>Backup the Oozie Server database on {{oozie-env/oozie_hostname}}</message>
+ </task>
+ </execute-stage>
+
+ </group>
+
<group name="ZOOKEEPER" title="Zookeeper">
<service name="ZOOKEEPER">
<component>ZOOKEEPER_SERVER</component>
@@ -265,13 +296,6 @@
<service name="HDFS">
<component name="NAMENODE">
- <pre-upgrade>
- <task xsi:type="execute" hosts="master">
- <script>scripts/namenode.py</script>
- <function>prepare_rolling_upgrade</function>
- </task>
- </pre-upgrade>
-
<upgrade>
<task xsi:type="restart" />
</upgrade>
@@ -338,12 +362,6 @@
<service name="HBASE">
<component name="HBASE_MASTER">
- <pre-upgrade>
- <task xsi:type="execute" hosts="master">
- <script>scripts/hbase_upgrade.py</script>
- <function>snapshot</function>
- </task>
- </pre-upgrade>
<upgrade>
<task xsi:type="restart" />
</upgrade>
@@ -390,9 +408,6 @@
<component name="HIVE_METASTORE">
<pre-upgrade>
<task xsi:type="manual">
- <message>Backup the Hive Metastore database located on the following host(s): {{hosts.all}}</message>
- </task>
- <task xsi:type="manual">
<message>Consult the README documentation at /usr/hdp/{{version}}/hive/scripts/metastore/upgrade to correctly upgrade the Hive Metastore database. This database upgrade should be performed on the following host(s): {{hosts.all}}</message>
</task>
</pre-upgrade>
@@ -476,11 +491,6 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
- <pre-upgrade>
- <task xsi:type="manual">
- <message>Backup the Oozie Server database on {{oozie-env/oozie_hostname}}</message>
- </task>
- </pre-upgrade>
<upgrade>
<task xsi:type="restart" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a4d96450/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
index 00ba387..976eb53 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java
@@ -231,11 +231,11 @@ public class UpgradeResourceProviderTest {
List<Stage> stages = am.getRequestStatus(requests.get(0).longValue());
- assertEquals(9, stages.size());
+ assertEquals(10, stages.size());
List<HostRoleCommand> tasks = am.getRequestTasks(requests.get(0).longValue());
// same number of tasks as stages here
- assertEquals(9, tasks.size());
+ assertEquals(10, tasks.size());
Set<Long> slaveStageIds = new HashSet<Long>();
@@ -322,7 +322,7 @@ public class UpgradeResourceProviderTest {
ResourceProvider upgradeItemResourceProvider = new UpgradeItemResourceProvider(amc);
resources = upgradeItemResourceProvider.getResources(request, predicate);
- assertEquals(2, resources.size());
+ assertEquals(3, resources.size());
res = resources.iterator().next();
assertNotNull(res.getPropertyValue("UpgradeItem/status"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/a4d96450/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 6cd7c9f..396a91c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -151,6 +151,7 @@ public class UpgradeHelperTest {
assertEquals(postGroup.items.get(2).getText(), "Save Cluster State");
assertEquals(postGroup.items.get(2).getType(), StageWrapper.Type.SERVER_SIDE_ACTION);
+ assertEquals(4, groups.get(0).items.size());
assertEquals(6, groups.get(1).items.size());
assertEquals(8, groups.get(2).items.size());
assertEquals(8, groups.get(3).items.size());
@@ -238,6 +239,7 @@ public class UpgradeHelperTest {
assertEquals("Save Cluster State", postGroup.items.get(2).getText());
assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
+ assertEquals(4, groups.get(0).items.size());
assertEquals(2, groups.get(1).items.size());
assertEquals(8, groups.get(2).items.size());
assertEquals(7, groups.get(3).items.size());
http://git-wip-us.apache.org/repos/asf/ambari/blob/a4d96450/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
index 072a8b4..3dd7f2f 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
@@ -25,6 +25,11 @@
<message>Foo</message>
</task>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HIVE">
+ <task xsi:type="manual">
+ <message>Back stuff up.</message>
+ </task>
+ </execute-stage>
<execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS">
<task xsi:type="execute">
<command>ls</command>