You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/02/16 18:38:31 UTC
ambari git commit: AMBARI-15057. Oozie untar and prepare-war should
be only done exclusively in preupload.py (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.2 6f4f30e39 -> b6c8e5dce
AMBARI-15057. Oozie untar and prepare-war should be only done exclusively in preupload.py (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6c8e5dc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6c8e5dc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6c8e5dc
Branch: refs/heads/branch-2.2
Commit: b6c8e5dce1ce807b62fe245c3759b62de616a194
Parents: 6f4f30e
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Feb 16 19:37:37 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Feb 16 19:38:22 2016 +0200
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 11 +---
.../main/resources/scripts/Ambaripreupload.py | 59 ++++++++++++++++----
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 33 ++---------
3 files changed, 57 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c8e5dc/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index be4a191..1701bbd 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -17,7 +17,6 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import hashlib
import os
from resource_management.core.resources.service import ServiceConfig
@@ -228,11 +227,7 @@ def prepare_war():
if run_prepare_war:
# Time-consuming to run
- Execute(command,
- user=params.oozie_user
- )
-
- return_code, output = shell.call(command, user=params.oozie_user, logoutput=False, quiet=False)
+ return_code, output = shell.call(command, user=params.oozie_user)
if output is None:
output = ""
@@ -273,8 +268,7 @@ def oozie_server_specific():
)
hashcode_file = format("{oozie_home}/.hashcode")
- hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest()
- skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]")
+ skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
@@ -313,7 +307,6 @@ def oozie_server_specific():
prepare_war()
File(hashcode_file,
- content = hashcode,
mode = 0644,
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c8e5dc/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 5a20698..61db286 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -26,7 +26,6 @@ sys.path.append("/usr/lib/python2.6/site-packages")
import glob
from logging import thread
import re
-import hashlib
import tempfile
import time
import functools
@@ -143,6 +142,7 @@ with Environment() as env:
hdfs_site = ConfigDictionary({'dfs.webhdfs.enabled':False,
})
fs_default = get_fs_root()
+ oozie_secure = ''
oozie_env_sh_template = \
'''
#!/bin/bash
@@ -232,14 +232,13 @@ with Environment() as env:
source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
-
-
env.set_params(params)
hadoop_conf_dir = params.hadoop_conf_dir
oozie_libext_dir = format("/usr/hdp/{hdp_version}/oozie/libext")
oozie_home=format("/usr/hdp/{hdp_version}/oozie")
oozie_setup_sh=format("/usr/hdp/{hdp_version}/oozie/bin/oozie-setup.sh")
+ oozie_setup_sh_current="/usr/hdp/current/oozie-server/bin/oozie-setup.sh"
oozie_tmp_dir = "/var/tmp/oozie"
configure_cmds = []
configure_cmds.append(('tar','-xvf', oozie_home + '/oozie-sharelib.tar.gz','-C', oozie_home))
@@ -254,22 +253,62 @@ with Environment() as env:
)
hashcode_file = format("{oozie_home}/.hashcode")
- hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest()
- skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]")
+ skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
Execute( configure_cmds,
not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
- Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war"),
- user = params.oozie_user,
- not_if = format("{no_op_test} || {skip_recreate_sharelib}")
- )
+
File(hashcode_file,
- content = hashcode,
mode = 0644,
)
+
+ ###############################################
+ # PREPARE-WAR [BEGIN]
+ ###############################################
+ prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
+
+ # DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE.
+ # Oozie tmp dir should be /var/tmp/oozie and is already created by a function above.
+ command = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure} ")
+ command_to_file = format("cd {oozie_tmp_dir} && {oozie_setup_sh_current} prepare-war {oozie_secure} ")
+
+ run_prepare_war = False
+ if os.path.exists(prepare_war_cmd_file):
+ cmd = ""
+ with open(prepare_war_cmd_file, "r") as f:
+ cmd = f.readline().strip()
+ if command_to_file != cmd:
+ run_prepare_war = True
+ Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} has contents which differ.\n" \
+ "Expected: {command_to_file}.\nActual: {cmd}."))
+ else:
+ run_prepare_war = True
+ Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing."))
+
+ if run_prepare_war:
+ # Time-consuming to run
+ return_code, output = shell.call(command, user=params.oozie_user)
+ if output is None:
+ output = ""
+
+ if return_code != 0 or "New Oozie WAR file with added".lower() not in output.lower():
+ message = "Unexpected Oozie WAR preparation output {0}".format(output)
+ Logger.error(message)
+ raise Fail(message)
+
+ # Generate marker file
+ File(prepare_war_cmd_file,
+ content=command_to_file,
+ mode=0644,
+ )
+ else:
+ Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists."))
+ ###############################################
+ # PREPARE-WAR END [BEGIN]
+ ###############################################
oozie_shared_lib = format("/usr/hdp/{hdp_version}/oozie/share")
oozie_user = 'oozie'
oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c8e5dc/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 826a004..3bb4b30 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -25,14 +25,9 @@ from resource_management.core import shell
from resource_management.core.exceptions import Fail
from resource_management.libraries import functions
from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
-import hashlib
import tempfile
-md5_mock = MagicMock()
-md5_mock.hexdigest.return_value = "abc123hash"
-
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
-@patch.object(hashlib, "md5", new=MagicMock(return_value=md5_mock))
@patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={}))
@patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp"))
class TestOozieServer(RMFTestCase):
@@ -219,7 +214,7 @@ class TestOozieServer(RMFTestCase):
recursive = True,
)
self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
- not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
+ not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
sudo = True,
)
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
@@ -255,15 +250,11 @@ class TestOozieServer(RMFTestCase):
not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
- user = "oozie"
- )
self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
mode = 0644,
)
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
- content = 'abc123hash',
mode = 0644,
)
self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),
@@ -435,9 +426,9 @@ class TestOozieServer(RMFTestCase):
recursive = True,
)
self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
- not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
- sudo = True,
- )
+ not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
+ sudo = True,
+ )
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
sudo = True,
@@ -472,15 +463,11 @@ class TestOozieServer(RMFTestCase):
self.assertResourceCalled('Execute', 'ambari-sudo.sh chown oozie:hadoop /usr/lib/oozie/libext/falcon-oozie-el-extension-*.jar',
not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
- user = 'oozie',
- )
self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
mode = 0644,
)
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
- content = 'abc123hash',
mode = 0644,
)
self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'),
@@ -804,7 +791,7 @@ class TestOozieServer(RMFTestCase):
recursive = True,
)
self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
- not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
+ not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
sudo = True,
)
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
@@ -826,15 +813,11 @@ class TestOozieServer(RMFTestCase):
not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
- user="oozie")
-
self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
mode = 0644,
)
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
- content = 'abc123hash',
mode = 0644,
)
self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),
@@ -991,7 +974,7 @@ class TestOozieServer(RMFTestCase):
recursive = True,
)
self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
- not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
+ not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
sudo = True,
)
self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
@@ -1013,15 +996,11 @@ class TestOozieServer(RMFTestCase):
not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure',
- user="oozie")
-
self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure',
mode = 0644,
)
self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
- content = 'abc123hash',
mode = 0644,
)
self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),