You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2018/08/17 20:22:18 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-24501]
- Update UpgradeSummary to Include Mpack Information (#2110)
This is an automated email from the ASF dual-hosted git repository.
jonathanhurley pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new baada3d [AMBARI-24501] - Update UpgradeSummary to Include Mpack Information (#2110)
baada3d is described below
commit baada3d72c851b5e80b5f1e998d7390dc8157e37
Author: Jonathan Hurley <jo...@apache.org>
AuthorDate: Fri Aug 17 16:22:15 2018 -0400
[AMBARI-24501] - Update UpgradeSummary to Include Mpack Information (#2110)
---
.../libraries/functions/upgrade_summary.py | 144 +++++++++++++--------
.../apache/ambari/server/state/UpgradeContext.java | 59 ++++-----
.../src/test/python/TestUpgradeSummary.py | 120 +++++++++--------
3 files changed, 170 insertions(+), 153 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py b/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py
index e6da97f..7bc8c20 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/upgrade_summary.py
@@ -22,95 +22,102 @@ from collections import namedtuple
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.constants import Direction
-UpgradeSummary = namedtuple("UpgradeSummary", "type direction orchestration is_revert services is_downgrade_allowed is_switch_bits associated_stack associated_version")
-UpgradeServiceSummary = namedtuple("UpgradeServiceSummary", "service_name source_stack source_version target_stack target_version")
+UpgradeSummary = namedtuple("UpgradeSummary", "direction is_revert service_groups")
+UpgradeServiceGroupSummary = namedtuple("UpgradeServiceGroupSummary", "type service_group_id service_group_name source_mpack_id target_mpack_id source_stack target_stack source_mpack_version target_mpack_version services")
+UpgradeServiceSummary = namedtuple("UpgradeServiceSummary", "service_name source_version target_version components")
+UpgradeComponentSummary = namedtuple("UpgradeComponentSummary", "component_name source_version target_version")
-def get_source_stack(service_name):
+
+def get_upgrade_summary():
"""
- Gets the source stack (from) version of a service participating in an upgrade. If there is no
- upgrade or the specific service is not participating, this will return None.
- :param service_name: the service name to check for, or None to extract it from the command
- :return: the stack that the service is upgrading from or None if there is no upgrade or
- the service is not included in the upgrade.
+ Gets a summary of an upgrade in progress, including type, direction, orchestration and from/to
+ versions.
"""
- service_summary = _get_service_summary(service_name)
- if service_summary is None:
+ config = Script.get_config()
+ if "upgradeSummary" not in config or not config["upgradeSummary"]:
return None
- return service_summary.source_stack
+ upgrade_summary = config["upgradeSummary"]
+
+ service_group_summary_dict = {}
+ for service_group_name, service_group_summary_json in upgrade_summary["serviceGroups"].iteritems():
+ service_summary_dict = {}
+
+ service_group_summary = UpgradeServiceGroupSummary(type = service_group_summary_json["type"],
+ service_group_id = service_group_summary_json["serviceGroupId"],
+ service_group_name = service_group_summary_json["serviceGroupName"],
+ source_mpack_id = service_group_summary_json["sourceMpackId"],
+ target_mpack_id = service_group_summary_json["targetMpackId"],
+ source_stack = service_group_summary_json["sourceStack"],
+ target_stack = service_group_summary_json["targetStack"],
+ source_mpack_version = service_group_summary_json["sourceMpackVersion"],
+ target_mpack_version = service_group_summary_json["targetMpackVersion"],
+ services = service_summary_dict)
+ service_group_summary_dict[service_group_name] = service_group_summary
-def get_source_version(service_name = None, default_version=None):
+ for service_name, service_summary_json in service_group_summary_json["services"].iteritems():
+ component_summary_dict = {}
+
+ service_summary = UpgradeServiceSummary(service_name = service_name,
+ source_version = service_summary_json["sourceVersion"],
+ target_version = service_summary_json["targetVersion"], components = component_summary_dict)
+
+ service_summary_dict[service_name] = service_summary
+
+ for component_name, component_summary_json in service_summary_json["components"].iteritems():
+ component_summary = UpgradeComponentSummary(component_name = component_name,
+ source_version = component_summary_json["sourceVersion"],
+ target_version = component_summary_json["targetVersion"])
+ component_summary_dict[component_name] = component_summary
+
+ return UpgradeSummary(direction=upgrade_summary["direction"],
+ is_revert = upgrade_summary["isRevert"],
+ service_groups = service_group_summary_dict)
+
+
+def get_source_version(service_group_name = None, service_name = None, default_version=None):
"""
Gets the source (from) version of a service participating in an upgrade. If there is no
upgrade or the specific service is not participating, this will return None.
+ :param service_group_name: the service group name to check for, or None to extract it from the command
:param service_name: the service name to check for, or None to extract it from the command
:param default_version: if the version of the service can't be calculated, this optional
default value is returned
:return: the version that the service is upgrading from or None if there is no upgrade or
the service is not included in the upgrade.
"""
- service_summary = _get_service_summary(service_name)
+ service_summary = _get_service_summary(service_group_name, service_name)
if service_summary is None:
return default_version
return service_summary.source_version
-def get_target_version(service_name = None, default_version=None):
+def get_target_version(service_group_name = None, service_name = None, default_version=None):
"""
Gets the target (to) version of a service participating in an upgrade. If there is no
upgrade or the specific service is not participating, this will return None.
+ :param service_group_name: the service group name to check for, or None to extract it from the command
:param service_name: the service name to check for, or None to extract it from the command
:param default_version: if the version of the service can't be calculated, this optional
default value is returned
:return: the version that the service is upgrading to or None if there is no upgrade or
the service is not included in the upgrade.
"""
- service_summary = _get_service_summary(service_name)
+ service_summary = _get_service_summary(service_group_name, service_name)
if service_summary is None:
return default_version
return service_summary.target_version
-
-def get_upgrade_summary():
- """
- Gets a summary of an upgrade in progress, including type, direction, orchestration and from/to
- repository versions.
- """
- config = Script.get_config()
- if "upgradeSummary" not in config or not config["upgradeSummary"]:
- return None
-
- upgrade_summary = config["upgradeSummary"]
- service_summary_dict = {}
-
- service_summary = upgrade_summary["services"]
- for service_name, service_summary_json in service_summary.iteritems():
- service_summary = UpgradeServiceSummary(service_name = service_name,
- source_stack = service_summary_json["sourceStackId"],
- source_version = service_summary_json["sourceVersion"],
- target_stack = service_summary_json["targetStackId"],
- target_version = service_summary_json["targetVersion"])
-
- service_summary_dict[service_name] = service_summary
-
- return UpgradeSummary(type=upgrade_summary["type"], direction=upgrade_summary["direction"],
- orchestration=upgrade_summary["orchestration"], is_revert = upgrade_summary["isRevert"],
- services = service_summary_dict,
- is_downgrade_allowed=upgrade_summary["isDowngradeAllowed"],
- is_switch_bits=upgrade_summary["isSwitchBits"],
- associated_stack=upgrade_summary["associatedStackId"],
- associated_version = upgrade_summary["associatedVersion"])
-
-
-def get_downgrade_from_version(service_name = None):
+def get_downgrade_from_version(service_group_name = None, service_name = None):
"""
Gets the downgrade-from-version for the specificed service. If there is no downgrade or
the service isn't participating in the downgrade, then this will return None
+ :param service_group_name: the service group, or optionally onmitted to infer it from the command.
:param service_name: the service, or optionally onmitted to infer it from the command.
:return: the downgrade-from-version or None
"""
@@ -121,17 +128,40 @@ def get_downgrade_from_version(service_name = None):
if Direction.DOWNGRADE.lower() != upgrade_summary.direction.lower():
return None
- service_summary = _get_service_summary(service_name)
+ service_summary = _get_service_summary(service_group_name, service_name)
if service_summary is None:
return None
return service_summary.source_version
-def _get_service_summary(service_name):
+def _get_service_group_summary(service_group_name):
+ """
+ Gets the service group summary for the upgrade/downgrade for the given service group, or None if
+ the service group isn't participating.
+ :param service_group_name the service group name
+ :return: the service group summary or None
+ """
+ upgrade_summary = get_upgrade_summary()
+ if upgrade_summary is None:
+ return None
+
+ if service_group_name is None:
+ execution_command = Script.get_execution_command()
+ service_group_name = execution_command.get_servicegroup_name()
+
+ service_group_summary = upgrade_summary.service_groups
+ if service_group_name not in service_group_summary:
+ return None
+
+ return service_group_summary[service_group_name]
+
+
+def _get_service_summary(service_group_name, service_name):
"""
Gets the service summary for the upgrade/downgrade for the given service, or None if
the service isn't participating.
+ :param service_group_name the service group name
:param service_name: the service name
:return: the service summary or None
"""
@@ -139,12 +169,16 @@ def _get_service_summary(service_name):
if upgrade_summary is None:
return None
+ execution_command = Script.get_execution_command()
+
+ if service_group_name is None:
+ service_group_name = execution_command.get_servicegroup_name()
+
if service_name is None:
- config = Script.get_config()
- service_name = config['serviceName']
+ service_name = execution_command.get_module_name()
- service_summary = upgrade_summary.services
- if service_name not in service_summary:
+ service_group_summary = _get_service_group_summary(service_group_name)
+ if service_group_summary is None or service_name not in service_group_summary.services:
return None
- return service_summary[service_name]
+ return service_group_summary.services[service_name]
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
index 513a727..5d1cf4e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
@@ -637,9 +637,13 @@ public class UpgradeContext {
UpgradeServiceGroupSummary serviceGroupSummary = new UpgradeServiceGroupSummary();
serviceGroupSummary.type = m_type;
+ serviceGroupSummary.serviceGroupId = serviceGroup.getServiceGroupId();
+ serviceGroupSummary.serviceGroupName = serviceGroup.getServiceGroupName();
serviceGroupSummary.sourceMpackId = changeSummary.getSource().getResourceId();
+ serviceGroupSummary.sourceMpackVersion = changeSummary.getSource().getVersion();
serviceGroupSummary.sourceStack = changeSummary.getSource().getStackId().getStackId();
serviceGroupSummary.targetMpackId = changeSummary.getTarget().getRegistryId();
+ serviceGroupSummary.targetMpackVersion = changeSummary.getTarget().getVersion();
serviceGroupSummary.targetStack = changeSummary.getTarget().getStackId().getStackId();
serviceGroupSummary.services = new LinkedHashMap<>();
@@ -647,6 +651,7 @@ public class UpgradeContext {
for( ModuleVersionChange moduleVersionChange : changeSummary.getModuleVersionChanges() ) {
UpgradeServiceSummary upgradeServiceSummary = new UpgradeServiceSummary();
+ upgradeServiceSummary.serviceName = moduleVersionChange.getSource().getName();
upgradeServiceSummary.sourceVersion = moduleVersionChange.getSource().getVersion();
upgradeServiceSummary.targetVersion = moduleVersionChange.getTarget().getVersion();
upgradeServiceSummary.components = new LinkedHashMap<>();
@@ -654,6 +659,7 @@ public class UpgradeContext {
for( ModuleComponentVersionChange componentVersionChange : moduleVersionChange.getComponentChanges() ) {
UpgradeComponentSummary componentSummary = new UpgradeComponentSummary();
+ componentSummary.componentName = componentVersionChange.getSource().getName();
componentSummary.sourceVersion = componentVersionChange.getSource().getVersion();
componentSummary.targetVersion = componentVersionChange.getTarget().getVersion();
@@ -1208,6 +1214,12 @@ public class UpgradeContext {
@SerializedName("type")
public UpgradeType type;
+ @SerializedName("serviceGroupId")
+ public Long serviceGroupId;
+
+ @SerializedName("serviceGroupName")
+ public String serviceGroupName;
+
@SerializedName("sourceMpackId")
public long sourceMpackId;
@@ -1220,8 +1232,11 @@ public class UpgradeContext {
@SerializedName("targetStack")
public String targetStack;
- @SerializedName("downgradeAllowed")
- public boolean isDowngradeAllowed = true;
+ @SerializedName("sourceMpackVersion")
+ public String sourceMpackVersion;
+
+ @SerializedName("targetMpackVersion")
+ public String targetMpackVersion;
/**
* A mapping of service name to service summary information for services
@@ -1229,40 +1244,6 @@ public class UpgradeContext {
*/
@SerializedName("services")
public Map<String, UpgradeServiceSummary> services;
-
- /**
- * The ID of the repository associated with the upgrade. For an
- * {@link Direction#UPGRADE}, this is the target repository, for a
- * {@link Direction#DOWNGRADE} this was the repository being downgraded
- * from.
- */
- @SerializedName("associatedRepositoryId")
- public long associatedRepositoryId;
-
- /**
- * The ID of the repository associated with the upgrade. For an
- * {@link Direction#UPGRADE}, this is the target stack, for a
- * {@link Direction#DOWNGRADE} this was the stack that is being downgraded
- * from.
- */
- @SerializedName("associatedStackId")
- public String associatedStackId;
-
- /**
- * The ID of the repository associated with the upgrade. For an
- * {@link Direction#UPGRADE}, this is the target versopm, for a
- * {@link Direction#DOWNGRADE} this was the version that is being downgraded
- * from.
- */
- @SerializedName("associatedVersion")
- public String associatedVersion;
-
- /**
- * MAINT or PATCH upgrades are meant to just be switching the bits and no other
- * incompatible changes.
- */
- @SerializedName("isSwitchBits")
- public boolean isSwitchBits = false;
}
/**
@@ -1270,6 +1251,9 @@ public class UpgradeContext {
* service component upgrade information during an upgrade.
*/
public static class UpgradeServiceSummary {
+ @SerializedName("serviceName")
+ public String serviceName;
+
@SerializedName("sourceVersion")
public String sourceVersion;
@@ -1288,6 +1272,9 @@ public class UpgradeContext {
* the component source and target versions during an upgrade.
*/
public static class UpgradeComponentSummary {
+ @SerializedName("componentName")
+ public String componentName;
+
@SerializedName("sourceVersion")
public String sourceVersion;
diff --git a/ambari-server/src/test/python/TestUpgradeSummary.py b/ambari-server/src/test/python/TestUpgradeSummary.py
index e2bc2c5..2e16482 100644
--- a/ambari-server/src/test/python/TestUpgradeSummary.py
+++ b/ambari-server/src/test/python/TestUpgradeSummary.py
@@ -28,7 +28,7 @@ Logger.initialize_logger()
class TestUpgradeSummary(TestCase):
- def test_get_stack_feature_version_missing_params(self):
+ def test_upgrade_summary(self):
"""
Tests that simple upgrade information can be extracted from JSON
:return:
@@ -39,17 +39,18 @@ class TestUpgradeSummary(TestCase):
summary = upgrade_summary.get_upgrade_summary()
self.assertEqual(False, summary.is_revert)
self.assertEqual("UPGRADE", summary.direction)
- self.assertEqual("STANDARD", summary.orchestration)
- self.assertEqual("rolling_upgrade", summary.type)
- services = summary.services
- self.assertEqual("2.4.0.0-1234", services["HDFS"].source_version)
- self.assertEqual("2.5.9.9-9999", services["HDFS"].target_version)
+ service_groups = summary.service_groups
+ self.assertEqual("express_upgrade", service_groups["SG1"].type)
- self.assertEqual("2.4.0.0-1234", upgrade_summary.get_source_version("HDFS"))
- self.assertEqual("2.5.9.9-9999", upgrade_summary.get_target_version("HDFS"))
+ services = service_groups["SG1"].services
+ self.assertEqual("3.0.0.0-b1", services["HDFS"].source_version)
+ self.assertEqual("3.1.0.0-b1", services["HDFS"].target_version)
- self.assertTrue(upgrade_summary.get_downgrade_from_version("HDFS") is None)
+ self.assertEqual("3.0.0.0-b1", upgrade_summary.get_source_version(service_group_name = "SG1", service_name = "HDFS"))
+ self.assertEqual("3.1.0.0-b1", upgrade_summary.get_target_version(service_group_name = "SG1", service_name = "HDFS"))
+
+ self.assertTrue(upgrade_summary.get_downgrade_from_version(service_group_name = "SG1", service_name="HDFS") is None)
def test_get_downgrade_from_version(self):
@@ -60,8 +61,8 @@ class TestUpgradeSummary(TestCase):
command_json = TestUpgradeSummary._get_cluster_simple_downgrade_json()
Script.config = command_json
- self.assertTrue(upgrade_summary.get_downgrade_from_version("FOO") is None)
- self.assertEqual("2.5.9.9-9999", upgrade_summary.get_downgrade_from_version("HDFS"))
+ self.assertTrue(upgrade_summary.get_downgrade_from_version(service_group_name = "FOO", service_name = "BAR") is None)
+ self.assertEqual("3.1.0.0-b1", upgrade_summary.get_downgrade_from_version(service_group_name = "SG1", service_name = "HDFS"))
@staticmethod
@@ -72,35 +73,34 @@ class TestUpgradeSummary(TestCase):
"""
return {
"roleCommand":"ACTIONEXECUTE",
- "hostLevelParams": {
- "stack_name": "HDP",
- "stack_version": "2.4",
- },
- "commandParams": {
- "source_stack": "2.4",
- "target_stack": "2.5",
- "upgrade_direction": "upgrade",
- "version": "2.5.9.9-9999"
- },
"upgradeSummary": {
- "services":{
- "HDFS":{
- "sourceRepositoryId":1,
- "sourceStackId":"HDP-2.4",
- "sourceVersion":"2.4.0.0-1234",
- "targetRepositoryId":2,
- "targetStackId":"HDP-2.5",
- "targetVersion":"2.5.9.9-9999"
+ "serviceGroups":{
+ "SG1":{
+ "type":"express_upgrade",
+ "serviceGroupId": 1,
+ "serviceGroupName": "SG1",
+ "sourceMpackId": 50,
+ "targetMpackId": 100,
+ "sourceStack": "HDPCORE-1.0",
+ "targetStack": "HDPCORE-1.5",
+ "sourceMpackVersion": "1.0.0.0-b1",
+ "targetMpackVersion": "1.5.0.0-b1",
+ "services":{
+ "HDFS":{
+ "serviceName": "HDFS",
+ "sourceVersion":"3.0.0.0-b1",
+ "targetVersion":"3.1.0.0-b1",
+ "components": {
+ "componentName": "NAMENODE",
+ "sourceVersion": "3.0.0.0-b1",
+ "targetVersion":"3.1.0.0-b1",
+ }
+ }
+ }
}
},
"direction":"UPGRADE",
- "type":"rolling_upgrade",
- "isRevert":False,
- "orchestration":"STANDARD",
- "associatedStackId":"HDP-2.5",
- "associatedVersion":"2.5.9.9-9999",
- "isDowngradeAllowed": True,
- "isSwitchBits": False
+ "isRevert":False
}
}
@@ -112,34 +112,30 @@ class TestUpgradeSummary(TestCase):
"""
return {
"roleCommand":"ACTIONEXECUTE",
- "hostLevelParams": {
- "stack_name": "HDP",
- "stack_version": "2.4",
- },
- "commandParams": {
- "source_stack": "2.5",
- "target_stack": "2.4",
- "upgrade_direction": "downgrade",
- "version": "2.4.0.0-1234"
- },
"upgradeSummary": {
- "services":{
- "HDFS":{
- "sourceRepositoryId":2,
- "sourceStackId":"HDP-2.5",
- "sourceVersion":"2.5.9.9-9999",
- "targetRepositoryId":1,
- "targetStackId":"HDP-2.4",
- "targetVersion":"2.4.0.0-1234"
+ "serviceGroups":{
+ "SG1":{
+ "type": "express_upgrade",
+ "serviceGroupId": 1,
+ "serviceGroupName": "SG1",
+ "sourceMpackId": 100,
+ "targetMpackId": 50,
+ "sourceStack": "HDPCORE-1.5",
+ "targetStack": "HDPCORE-1.0",
+ "sourceMpackVersion": "1.5.0.0-b1",
+ "targetMpackVersion": "1.0.0.0-b1",
+ "services":{
+ "HDFS":{
+ "serviceName": "HDFS",
+ "sourceVersion":"3.1.0.0-b1",
+ "targetVersion":"3.0.0.0-b1",
+ "components": {
+ }
+ }
+ }
}
},
- "direction":"DOWNGRADE",
- "type":"rolling_upgrade",
- "isRevert":False,
- "orchestration":"STANDARD",
- "associatedStackId":"HDP-2.5",
- "associatedVersion":"2.5.9.9-9999",
- "isDowngradeAllowed": True,
- "isSwitchBits": False
+ "direction": "DOWNGRADE",
+ "isRevert": False
}
}