You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/07/24 15:27:10 UTC
[44/50] [abbrv] ambari git commit: Cross-stack upgrade,
Oozie restart fails with ext-2.2.zip missing error,
stack_tools.py is missing get_stack_name in __all__,
disable BigInsights in UI (Alejandro Fernandez via smohanty)
Cross-stack upgrade, Oozie restart fails with ext-2.2.zip missing error, stack_tools.py is missing get_stack_name in __all__, disable BigInsights in UI (Alejandro Fernandez via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eca55998
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eca55998
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eca55998
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: eca55998d38f6a8edbb156d84c6cc963c68e85a9
Parents: cfd7bb4
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Jul 21 12:07:34 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Jul 21 12:10:28 2017 -0700
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 44 ++++++++++++++++----
.../package/scripts/oozie_server_upgrade.py | 34 +++++++++------
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 32 ++++++++++----
3 files changed, 84 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca55998/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index d916d3b..3467ed2 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -215,7 +215,27 @@ def oozie_ownership():
group = params.user_group
)
-def oozie_server_specific():
+def get_oozie_ext_zip_source_paths(upgrade_type, params):
+ """
+ Get an ordered list of Oozie ext zip file paths from the source stack.
+ :param upgrade_type: Upgrade type will be None if not in the middle of a stack upgrade.
+ :param params: Expected to contain fields for ext_js_path, upgrade_direction, source_stack_name, and ext_js_file
+ :return: Source paths to use for Oozie extension zip file
+ """
+ # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip
+ paths = []
+ source_ext_js_path = params.ext_js_path
+ # Preferred location used by HDP and BigInsights 4.2.5
+ if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE:
+ source_ext_js_path = "/usr/share/" + params.source_stack_name.upper() + "-oozie/" + params.ext_js_file
+ paths.append(source_ext_js_path)
+
+ # Alternate location used by BigInsights 4.2.0 when migrating to another stack.
+ paths.append("/var/lib/oozie/" + params.ext_js_file)
+
+ return paths
+
+def oozie_server_specific(upgrade_type):
import params
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
@@ -249,13 +269,23 @@ def oozie_server_specific():
)
configure_cmds = []
- configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir))
- configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
+ # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
+ source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
+
+ # Copy the first oozie ext-2.2.zip file that is found.
+ # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
+ if source_ext_zip_paths is not None:
+ for source_ext_zip_path in source_ext_zip_paths:
+ if os.path.isfile(source_ext_zip_path):
+ configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
+ configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
+
+ Execute(configure_cmds,
+ not_if=no_op_test,
+ sudo=True,
+ )
+ break
- Execute( configure_cmds,
- not_if = no_op_test,
- sudo = True,
- )
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca55998/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index 402c7cb..3edb042 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -86,21 +86,31 @@ class OozieUpgrade(Script):
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
- # copy ext ZIP to libext dir
- oozie_ext_zip_file = params.ext_js_path
-
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file)
- if not os.path.isfile(oozie_ext_zip_file):
- raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
-
- Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
- Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
- Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
- File(oozie_ext_zip_target_path,
- mode=0644
- )
+ # Copy ext ZIP to libext dir
+ # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
+ source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(upgrade_type, params)
+
+ found_at_least_one_oozie_ext_file = False
+
+ # Copy the first oozie ext-2.2.zip file that is found.
+ # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
+ if source_ext_zip_paths is not None:
+ for source_ext_zip_path in source_ext_zip_paths:
+ if os.path.isfile(source_ext_zip_path):
+ found_at_least_one_oozie_ext_file = True
+ Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir))
+ Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True)
+ Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
+ File(oozie_ext_zip_target_path,
+ mode=0644
+ )
+ break
+
+ if not found_at_least_one_oozie_ext_file:
+ raise Fail("Unable to find any Oozie source extension files from the following paths {0}".format(source_ext_zip_paths))
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca55998/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 426c36a..5ef6ad9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -44,7 +44,9 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
- def test_configure_default(self, call_mocks):
+ @patch("os.path.isfile")
+ def test_configure_default(self, isfile_mock, call_mocks):
+ isfile_mock.return_value = True
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
classname = "OozieServer",
@@ -59,7 +61,10 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True]))
- def test_configure_default_mysql(self, call_mocks):
+ @patch("os.path.isfile")
+ def test_configure_default_mysql(self, isfile_mock, iscall_mocks):
+ # Mock call when checking if need to copy oozie ext-2.2.zip file
+ isfile_mock.side_effect = [True, False]
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
classname = "OozieServer",
@@ -281,7 +286,10 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True]))
- def test_configure_existing_sqla(self, call_mocks):
+ @patch("os.path.isfile")
+ def test_configure_existing_sqla(self, isfile_mock, call_mocks):
+ # Mock call when checking if need to copy oozie ext-2.2.zip file
+ isfile_mock.side_effect = [True, False]
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
classname = "OozieServer",
@@ -602,7 +610,10 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True]))
- def test_configure_secured(self, call_mocks):
+ @patch("os.path.isfile")
+ def test_configure_secured(self, isfile_mock, call_mocks):
+ # Mock call when checking if need to copy oozie ext-2.2.zip file
+ isfile_mock.side_effect = [True, False]
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
classname = "OozieServer",
@@ -616,8 +627,11 @@ class TestOozieServer(RMFTestCase):
self.assertNoMoreResources()
@patch.object(shell, "call")
+ @patch("os.path.isfile")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True]))
- def test_configure_secured_ha(self, call_mocks):
+ def test_configure_secured_ha(self, isfile_mock, call_mocks):
+ # Mock call when checking if need to copy oozie ext-2.2.zip file
+ isfile_mock.side_effects = [True, False]
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
config_file = "stacks/2.0.6/configs/secured.json"
@@ -650,7 +664,8 @@ class TestOozieServer(RMFTestCase):
@patch("os.path.isfile")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True]))
def test_start_secured(self, isfile_mock, call_mocks):
- isfile_mock.return_value = True
+ # Mock call when checking if need to copy oozie ext-2.2.zip file
+ isfile_mock.side_effects = [True, False]
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
classname = "OozieServer",
@@ -1133,7 +1148,10 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True]))
- def test_configure_default_hdp22(self, call_mocks):
+ @patch("os.path.isfile")
+ def test_configure_default_hdp22(self, isfile_mock, call_mocks):
+ # Mock call when checking if need to copy oozie ext-2.2.zip file
+ isfile_mock.side_effect = [True, False]
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
config_file = "stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f: