You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/12/01 04:14:46 UTC
ambari git commit: AMBARI-22568 - Oozie Fails To Restart During
Upgrade Because of Missing ExtJS Library (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 2a731ac96 -> 306b9a5ee
AMBARI-22568 - Oozie Fails To Restart During Upgrade Because of Missing ExtJS Library (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/306b9a5e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/306b9a5e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/306b9a5e
Branch: refs/heads/branch-2.6
Commit: 306b9a5eeb10d683e4f1ce70031e412b28c4a707
Parents: 2a731ac
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Nov 30 15:24:24 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Nov 30 23:14:38 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/constants.py | 11 +++++----
.../package/scripts/oozie_server_upgrade.py | 24 ++++++++++++++++----
.../HDP/2.0.6/properties/stack_features.json | 6 +++++
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 24 +++++++++++++++++---
4 files changed, 52 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/306b9a5e/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index b811861..212827c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -111,13 +111,14 @@ class StackFeature:
KAFKA_RANGER_PLUGIN_SUPPORT = "kafka_ranger_plugin_support"
YARN_RANGER_PLUGIN_SUPPORT = "yarn_ranger_plugin_support"
RANGER_SOLR_CONFIG_SUPPORT='ranger_solr_config_support'
- HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED="hive_interactive_atlas_hook_required"
- CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins'
- ATLAS_INSTALL_HOOK_PACKAGE_SUPPORT="atlas_install_hook_package_support"
- ATLAS_HDFS_SITE_ON_NAMENODE_HA='atlas_hdfs_site_on_namenode_ha'
- HIVE_INTERACTIVE_GA_SUPPORT='hive_interactive_ga'
+ HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED = "hive_interactive_atlas_hook_required"
+ CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT = 'core_site_for_ranger_plugins'
+ ATLAS_INSTALL_HOOK_PACKAGE_SUPPORT = "atlas_install_hook_package_support"
+ ATLAS_HDFS_SITE_ON_NAMENODE_HA = 'atlas_hdfs_site_on_namenode_ha'
+ HIVE_INTERACTIVE_GA_SUPPORT = 'hive_interactive_ga'
SECURE_RANGER_SSL_PASSWORD = "secure_ranger_ssl_password"
RANGER_KMS_SSL = "ranger_kms_ssl"
KAFKA_ACL_MIGRATION_SUPPORT = "kafka_acl_migration_support"
ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support"
KAFKA_EXTENDED_SASL_SUPPORT = "kafka_extended_sasl_support"
+ OOZIE_EXTJS_INCLUDED = "oozie_extjs_included"
http://git-wip-us.apache.org/repos/asf/ambari/blob/306b9a5e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index eb57c22..d3cecef 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -28,6 +28,7 @@ from resource_management.core.resources.system import File
from resource_management.libraries.functions import Direction
from resource_management.libraries.functions import format
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import lzo_utils
from resource_management.libraries.functions.oozie_prepare_war import prepare_war
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions import StackFeature
@@ -52,7 +53,8 @@ class OozieUpgrade(Script):
import params
# some stack versions don't need the lzo compression libraries
- target_version_needs_compression_libraries = params.version and check_stack_feature(StackFeature.LZO, params.version)
+ target_version_needs_compression_libraries = check_stack_feature(StackFeature.LZO,
+ params.version_for_stack_feature_checks)
# ensure the directory exists
Directory(params.oozie_libext_dir, mode = 0777)
@@ -66,6 +68,9 @@ class OozieUpgrade(Script):
# When a version is Installed, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
+ # ensure that the LZO files are installed for this version of Oozie
+ lzo_utils.install_lzo_if_needed()
+
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("{stack_root}/{version}/hadoop/lib")
@@ -86,6 +91,12 @@ class OozieUpgrade(Script):
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+ # ExtJS is used to build a working Oozie Web UI - without it, Oozie will startup and work
+ # but will not have a functioning user interface - Some stacks no longer ship ExtJS,
+ # so it's optional now. On an upgrade, we should make sure that if it's not found, that's OK
+ # However, if it is found on the system (from an earlier install) then it should be used
+ extjs_included = check_stack_feature(StackFeature.OOZIE_EXTJS_INCLUDED, params.version_for_stack_feature_checks)
+
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file)
@@ -104,14 +115,17 @@ class OozieUpgrade(Script):
Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir))
Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
- File(oozie_ext_zip_target_path,
- mode=0644
- )
+ File(oozie_ext_zip_target_path, mode=0644)
break
- if not found_at_least_one_oozie_ext_file:
+ # ExtJS was expected to the be on the system, but was not found
+ if extjs_included and not found_at_least_one_oozie_ext_file:
raise Fail("Unable to find any Oozie source extension files from the following paths {0}".format(source_ext_zip_paths))
+ # ExtJS is not expected, so it's OK - just log a warning
+ if not found_at_least_one_oozie_ext_file:
+ Logger.warning("Unable to find ExtJS in any of the following paths. The Oozie UI will not be available. Source Paths: {0}".format(source_ext_zip_paths))
+
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
http://git-wip-us.apache.org/repos/asf/ambari/blob/306b9a5e/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index 2109a5d..6d622ec 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -452,6 +452,12 @@
"name": "kafka_extended_sasl_support",
"description": "Support SASL PLAIN and GSSAPI",
"min_version": "2.6.5.0"
+ },
+ {
+ "name": "oozie_extjs_included",
+ "description": "ExtJS is included in the repository and automatically installed by Ambari",
+ "min_version": "2.2.0.0",
+ "max_version": "2.6.0.0"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/306b9a5e/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 4d94723..10a83d6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -28,7 +28,10 @@ from resource_management.libraries import functions
from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
import tempfile
-@patch.object(Script, 'format_package_name', new = MagicMock())
+def format_package_name_side_effect(name):
+ return name.replace("${stack_version}", "1_2_3_4")
+
+@patch.object(Script, 'format_package_name', new = MagicMock(side_effect=format_package_name_side_effect))
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
@patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={}))
@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp"))
@@ -1178,8 +1181,7 @@ class TestOozieServer(RMFTestCase):
@patch("glob.iglob")
@patch("shutil.copy2", new = MagicMock())
@patch("resource_management.core.sudo.path_isdir", new = MagicMock(return_value = True))
- def test_upgrade(self, glob_mock, remove_mock,
- isfile_mock, exists_mock, isdir_mock):
+ def test_upgrade(self, glob_mock, remove_mock, isfile_mock, exists_mock, isdir_mock):
def exists_mock_side_effect(path):
if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar':
@@ -1217,6 +1219,9 @@ class TestOozieServer(RMFTestCase):
sudo = True )
self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
+ self.assertResourceCalled('Package', ('lzo'), retry_count=5, retry_on_repo_unavailability= False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4'), retry_count = 5, retry_on_repo_unavailability = False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4-native'), retry_count = 5, retry_on_repo_unavailability = False)
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', mode = 0644)
@@ -1278,6 +1283,10 @@ class TestOozieServer(RMFTestCase):
self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
+ self.assertResourceCalled('Package', ('lzo'), retry_count=5, retry_on_repo_unavailability= False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4'), retry_count = 5, retry_on_repo_unavailability = False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4-native'), retry_count = 5, retry_on_repo_unavailability = False)
+
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', mode = 0644)
@@ -1346,6 +1355,10 @@ class TestOozieServer(RMFTestCase):
self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
+ self.assertResourceCalled('Package', ('lzo'), retry_count=5, retry_on_repo_unavailability= False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4'), retry_count = 5, retry_on_repo_unavailability = False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4-native'), retry_count = 5, retry_on_repo_unavailability = False)
+
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip', mode = 0644)
@@ -1383,6 +1396,7 @@ class TestOozieServer(RMFTestCase):
self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'oozie-server', u'2.2.0.0-0000'), sudo = True)
self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',mode = 0777)
+
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
self.assertResourceCalled('File', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip',mode = 0644)
@@ -1581,6 +1595,10 @@ class TestOozieServer(RMFTestCase):
self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
+ self.assertResourceCalled('Package', ('lzo'), retry_count=5, retry_on_repo_unavailability= False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4'), retry_count = 5, retry_on_repo_unavailability = False)
+ self.assertResourceCalled('Package', ('hadooplzo_1_2_3_4-native'), retry_count = 5, retry_on_repo_unavailability = False)
+
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)