You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/07/09 22:31:13 UTC
ambari git commit: AMBARI-12362. Manual upgrade fails - YARN RM
restart failed due to yarn node labels error.
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 6f6ffd051 -> d25236824
AMBARI-12362. Manual upgrade fails - YARN RM restart failed due to yarn node labels error.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2523682
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2523682
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2523682
Branch: refs/heads/branch-2.1
Commit: d25236824497fa7668e5083de0c7a81d12967cc3
Parents: 6f6ffd0
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Jul 9 13:23:28 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Jul 9 13:25:17 2015 -0700
----------------------------------------------------------------------
ambari-server/src/main/python/upgradeHelper.py | 15 ++++-
.../catalog/UpgradeCatalog_2.1_to_2.3.json | 17 ++++-
.../catalog/UpgradeCatalog_2.2_to_2.3.json | 17 ++++-
.../src/test/python/TestUpgradeHelper.py | 65 +++++++++++++++++++-
4 files changed, 108 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2523682/ambari-server/src/main/python/upgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/upgradeHelper.py b/ambari-server/src/main/python/upgradeHelper.py
index b2e0fa4..b5f123e 100644
--- a/ambari-server/src/main/python/upgradeHelper.py
+++ b/ambari-server/src/main/python/upgradeHelper.py
@@ -66,6 +66,10 @@ Example:
"value": "{TEMPLATE_TAG}",
"template": "yes",
"required-services": ["HDFS", "YARN"]
+ },
+ "test_property": {
+ "value": "new value",
+ "value-required": "old value" (optional, property would be set if the required value is present)
}
}
},
@@ -287,6 +291,7 @@ class CatConst(Const):
ITEMS_TAG = "items"
TYPE_TAG = "type"
TRUE_TAG = "yes"
+ VALUE_REQUIRED_TAG = "value-required"
STACK_PROPERTIES_MAPPING_LIST_TAG = "property-mapping"
VALUE_TEMPLATE_TAG = "template"
SEARCH_PATTERN = "(\{[^\{\}]+\})" # {XXXXX}
@@ -520,10 +525,12 @@ class UpgradeCatalog(object):
properties[catalog_item_name] = catalog_property_item[CatConst.PROPERTY_VALUE_TAG]
return properties
- def __can_handler_execute(self, catalog_options, property_item):
+ def __can_handler_execute(self, catalog_options, catalog_property_item, property_item, properties):
"""
:type catalog_options dict
+ :type catalog_property_item str
:type property_item dict
+ :type properties dict
"""
can_process = True
@@ -541,6 +548,10 @@ class UpgradeCatalog(object):
if required_list is not None:
can_process = can_process and is_services_exists(required_list)
+ if CatConst.VALUE_REQUIRED_TAG in property_item and property_item[CatConst.VALUE_REQUIRED_TAG] is not None and\
+ CatConst.PROPERTY_VALUE_TAG in property_item and catalog_property_item in properties:
+ can_process = properties[catalog_property_item] == property_item[CatConst.VALUE_REQUIRED_TAG]
+
return can_process
def process_simple_transformations(self, name, properties):
@@ -560,7 +571,7 @@ class UpgradeCatalog(object):
catalog_item = self._properties_catalog[name]
for catalog_property_item in catalog_item.keys():
catalog_options = self.options[name] if name in self.options else {}
- if self.__can_handler_execute(catalog_options, catalog_item[catalog_property_item]):
+ if self.__can_handler_execute(catalog_options, catalog_property_item, catalog_item[catalog_property_item], properties):
for handler in tag_handlers:
handler(catalog_property_item, catalog_item[catalog_property_item], properties)
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2523682/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
index c1c7970..1304c5a 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
@@ -16,6 +16,10 @@
"yarn-site": {
"merged-copy": "yes"
},
+ "capacity-scheduler": {
+ "merged-copy": "yes",
+ "required-services": ["YARN"]
+ },
"mapred-site": {
"merged-copy": "yes"
},
@@ -328,6 +332,17 @@
"yarn.timeline-service.recovery.enabled": "true",
"yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
"yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
+ },
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.root.default-node-label-expression": "",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {
+ "value": "10",
+ "value-required": "-1"
+ },
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {
+ "value": "100",
+ "value-required": "-1"
+ }
}
},
"property-mapping": {
@@ -344,4 +359,4 @@
}
}
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2523682/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
index 463b98e..875b271 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
@@ -35,6 +35,10 @@
"merged-copy": "yes",
"required-services": ["YARN"]
},
+ "capacity-scheduler": {
+ "merged-copy": "yes",
+ "required-services": ["YARN"]
+ },
"kafka-broker": {
"merged-copy": "yes",
"required-services": ["KAFKA"]
@@ -254,6 +258,17 @@
"yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
"yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
},
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.root.default-node-label-expression": "",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {
+ "value": "10",
+ "value-required": "-1"
+ },
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {
+ "value": "100",
+ "value-required": "-1"
+ }
+ },
"kafka-broker": {
"listeners": {
"value": "{KAFKA_LISTENERS}",
@@ -1693,4 +1708,4 @@
}
}
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2523682/ambari-server/src/test/python/TestUpgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestUpgradeHelper.py b/ambari-server/src/test/python/TestUpgradeHelper.py
index 835f39a..4956d9a 100644
--- a/ambari-server/src/test/python/TestUpgradeHelper.py
+++ b/ambari-server/src/test/python/TestUpgradeHelper.py
@@ -881,8 +881,6 @@ class TestUpgradeHelper(TestCase):
self.assertEqual(1, open_mock.call_count)
-
-
def test_report_formatter(self):
file = StringIO()
cfg_item = self.catalog_cfg_type
@@ -921,5 +919,68 @@ class TestUpgradeHelper(TestCase):
self.assertEqual(expected_output, file.getvalue())
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ def test_conditional_replace(self, get_config_resp_all_mock):
+ test_catalog = """
+ {
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "1.0",
+ "target-version": "1.1",
+ "options": {
+ "config-types":{
+ "test": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "test": {
+ "test": {
+ "value": "10",
+ "value-required": "-1"
+ },
+ "test2": {
+ "value": "10",
+ "value-required": "-2"
+ }
+ }
+ },
+ "property-mapping": {
+ }
+ }
+ ]
+ }
+ """
+
+ expected_properties = {"test":"10", "test2":"15"}
+
+ old_opt = upgradeHelper.Options.OPTIONS
+ options = lambda: ""
+ options.from_stack = "1.0"
+ options.to_stack = "1.1"
+ options.upgrade_json = ""
+
+ upgradeHelper.Options.OPTIONS = options
+ upgradeHelper.Options.SERVICES = [self.required_service]
+ get_config_resp_all_mock.return_value = {
+ "test": {
+ "properties": {"test":"-1", "test2":"15"}
+ }
+ }
+
+ ucf = UpgradeCatalogFactoryMock(test_catalog)
+ scf = upgradeHelper.ServerConfigFactory()
+
+ cfg = scf.get_config("test")
+ ucfg = ucf.get_catalog("1.0", "1.1")
+
+ cfg.merge(ucfg)
+ upgradeHelper.Options.OPTIONS = old_opt
+
+ self.assertEqual(expected_properties, cfg.properties)
+
if __name__ == "__main__":
unittest.main()