You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/03/22 16:07:09 UTC
ambari git commit: AMBARI-14863. Oozie should update war after adding
Falcon (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 3508a7ef6 -> 8bd43711a
AMBARI-14863. Oozie should update war after adding Falcon (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8bd43711
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8bd43711
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8bd43711
Branch: refs/heads/trunk
Commit: 8bd43711a284b521af0a883ee5b7b77e3471e0ed
Parents: 3508a7e
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Mar 22 17:07:08 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Mar 22 17:07:08 2016 +0200
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 31 ++++++++++++++++++--
.../main/resources/scripts/Ambaripreupload.py | 28 ++++++++++++++++--
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 30 +++++++++++++++++++
3 files changed, 84 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8bd43711/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index c503a38..5587380 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -26,6 +26,7 @@ from resource_management.core.source import InlineTemplate
from resource_management.core.source import Template
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.libraries.script.script import Script
from resource_management.core.resources.packaging import Package
@@ -199,12 +200,15 @@ def oozie_ownership():
def prepare_war():
"""
- Attempt to call prepare-war command if the marker file doesn't exist or its content doesn't equal the expected command.
- The marker file is stored in <stack-root>/current/oozie-server/.prepare_war_cmd
+ Attempt to call prepare-war command if the marker files don't exist or their content doesn't equal the expected.
+ The marker file for a command is stored in <stack-root>/current/oozie-server/.prepare_war_cmd
+ The marker file for a content of libext folder is stored in <stack-root>/current/oozie-server/.war_libext_content
"""
import params
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
+ libext_content_file = format("{oozie_home}/.war_libext_content")
+ list_libext_command = format("ls -l {oozie_libext_dir}") + " | awk '{print $9, $5}' | awk 'NF > 0'"
# DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE.
# Oozie tmp dir should be /var/tmp/oozie and is already created by a function above.
@@ -225,6 +229,23 @@ def prepare_war():
run_prepare_war = True
Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing."))
+ return_code, libext_content, error_output = get_user_call_output(list_libext_command, user=params.oozie_user)
+ libext_content = libext_content.strip()
+
+ if run_prepare_war == False:
+ if os.path.exists(libext_content_file):
+ old_content = ""
+ with open(libext_content_file, "r") as f:
+ old_content = f.read().strip()
+
+ if libext_content != old_content:
+ run_prepare_war = True
+ Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} has contents which differ.\n" \
+ "Content of the folder {oozie_libext_dir} changed."))
+ else:
+ run_prepare_war = True
+ Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} is missing."))
+
if run_prepare_war:
# Time-consuming to run
return_code, output = shell.call(command, user=params.oozie_user)
@@ -236,11 +257,15 @@ def prepare_war():
Logger.error(message)
raise Fail(message)
- # Generate marker file
+ # Generate marker files
File(prepare_war_cmd_file,
content=command,
mode=0644,
)
+ File(libext_content_file,
+ content=libext_content,
+ mode=0644,
+ )
else:
Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists."))
http://git-wip-us.apache.org/repos/asf/ambari/blob/8bd43711/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 941539c..cbec3cf 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -40,6 +40,7 @@ from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
from resource_management import Script
@@ -312,6 +313,8 @@ with Environment() as env:
# PREPARE-WAR [BEGIN]
###############################################
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
+ libext_content_file = format("{oozie_home}/.war_libext_content")
+ list_libext_command = format("ls -l {oozie_libext_dir}") + " | awk '{print $9, $5}' | awk 'NF > 0'"
# DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE.
# Oozie tmp dir should be /var/tmp/oozie and is already created by a function above.
@@ -332,6 +335,23 @@ with Environment() as env:
run_prepare_war = True
Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing."))
+ return_code, libext_content, error_output = get_user_call_output(list_libext_command, user=params.oozie_user)
+ libext_content = libext_content.strip()
+
+ if run_prepare_war == False:
+ if os.path.exists(libext_content_file):
+ old_content = ""
+ with open(libext_content_file, "r") as f:
+ old_content = f.read().strip()
+
+ if libext_content != old_content:
+ run_prepare_war = True
+ Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} has contents which differ.\n" \
+ "Content of the folder {oozie_libext_dir} changed."))
+ else:
+ run_prepare_war = True
+ Logger.info(format("Will run prepare war cmd since marker file {libext_content_file} is missing."))
+
if run_prepare_war:
# Time-consuming to run
return_code, output = shell.call(command, user=params.oozie_user)
@@ -343,11 +363,15 @@ with Environment() as env:
Logger.error(message)
raise Fail(message)
- # Generate marker file
+ # Generate marker files
File(prepare_war_cmd_file,
content=command_to_file,
mode=0644,
- )
+ )
+ File(libext_content_file,
+ content=libext_content,
+ mode=0644,
+ )
else:
Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists."))
###############################################
http://git-wip-us.apache.org/repos/asf/ambari/blob/8bd43711/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 407bb2f..ba61b3d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -40,6 +40,8 @@ class TestOozieServer(RMFTestCase):
self.maxDiff = None
@patch.object(shell, "call")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
def test_configure_default(self, call_mocks):
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
@@ -54,6 +56,8 @@ class TestOozieServer(RMFTestCase):
self.assertNoMoreResources()
@patch.object(shell, "call")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True]))
def test_configure_default_mysql(self, call_mocks):
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
@@ -261,6 +265,10 @@ class TestOozieServer(RMFTestCase):
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
mode = 0644,
)
+ self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content',
+ content = 'ext-2.2.zip',
+ mode = 0644,
+ )
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
mode = 0644,
)
@@ -271,6 +279,8 @@ class TestOozieServer(RMFTestCase):
)
@patch.object(shell, "call")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True]))
def test_configure_existing_sqla(self, call_mocks):
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
@@ -482,6 +492,10 @@ class TestOozieServer(RMFTestCase):
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
mode = 0644,
)
+ self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content',
+ content = 'ext-2.2.zip',
+ mode = 0644,
+ )
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
mode = 0644,
)
@@ -494,6 +508,8 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch("os.path.isfile")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
def test_start_default(self, isfile_mock, call_mocks):
isfile_mock.return_value = True
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
@@ -572,6 +588,8 @@ class TestOozieServer(RMFTestCase):
self.assertNoMoreResources()
@patch.object(shell, "call")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
def test_configure_secured(self, call_mocks):
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
@@ -587,6 +605,8 @@ class TestOozieServer(RMFTestCase):
@patch.object(shell, "call")
@patch("os.path.isfile")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
def test_start_secured(self, isfile_mock, call_mocks):
isfile_mock.return_value = True
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
@@ -848,6 +868,10 @@ class TestOozieServer(RMFTestCase):
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
mode = 0644,
)
+ self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content',
+ content = 'ext-2.2.zip',
+ mode = 0644,
+ )
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
mode = 0644,
)
@@ -1039,6 +1063,10 @@ class TestOozieServer(RMFTestCase):
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure',
mode = 0644,
)
+ self.assertResourceCalled('File', '/usr/lib/oozie/.war_libext_content',
+ content = 'ext-2.2.zip',
+ mode = 0644,
+ )
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
mode = 0644,
)
@@ -1049,6 +1077,8 @@ class TestOozieServer(RMFTestCase):
)
@patch.object(shell, "call")
+ @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0, 'ext-2.2.zip', '')))
+ @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
def test_configure_default_hdp22(self, call_mocks):
call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
config_file = "stacks/2.0.6/configs/default.json"