You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2016/12/27 10:44:46 UTC
ambari git commit: AMBARI-19297 Decrease time it takes to run Oozie
Service Check (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk b69ef96b0 -> eca4e54ee
AMBARI-19297 Decrease time it takes to run Oozie Service Check (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eca4e54e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eca4e54e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eca4e54e
Branch: refs/heads/trunk
Commit: eca4e54ee88bccf2ea8342d421dac5362fc6c3a3
Parents: b69ef96
Author: Dmytro Sen <ds...@apache.org>
Authored: Tue Dec 27 12:42:31 2016 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Tue Dec 27 12:42:41 2016 +0200
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 8 ++++++++
.../OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh | 11 ++++++-----
.../package/files/prepareOozieHdfsDirectories.sh | 15 ++++++++-------
.../OOZIE/4.0.0.2.0/package/scripts/params_linux.py | 2 ++
.../OOZIE/4.0.0.2.0/package/scripts/service_check.py | 8 ++++----
.../stacks/2.0.6/OOZIE/test_oozie_service_check.py | 3 ++-
.../python/stacks/2.0.6/OOZIE/test_service_check.py | 4 ++--
7 files changed, 32 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml
index dbc781f..7748176 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml
@@ -223,4 +223,12 @@ export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-{{architecture}}-64
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>service_check_job_name</name>
+ <value>no-op</value>
+ <description>
+ Job name from Oozie examples that will be executed at each Oozie service check action.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh
index 71a0c85..60716ae 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/oozieSmoke2.sh
@@ -29,10 +29,11 @@ export oozie_examples_dir=$6
export hadoop_conf_dir=$7
export hadoop_bin_dir=$8
export smoke_test_user=$9
-export security_enabled=${10}
-export smoke_user_keytab=${11}
-export kinit_path_local=${12}
-export smokeuser_principal=${13}
+export job_name=${10}
+export security_enabled=${11}
+export smoke_user_keytab=${12}
+export kinit_path_local=${13}
+export smokeuser_principal=${14}
function checkOozieJobStatus {
local job_id=$1
@@ -74,7 +75,7 @@ else
kinitcmd=""
fi
-cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/map-reduce/job.properties -run"
+cmd="${kinitcmd}source ${oozie_conf_dir}/oozie-env.sh ; ${oozie_bin_dir}/oozie -Doozie.auth.token.cache=false job -oozie $OOZIE_SERVER -config $oozie_examples_dir/examples/apps/${job_name}/job.properties -run"
echo $cmd
job_info=`/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "$cmd" | grep "job:"`
job_id="`echo $job_info | cut -d':' -f2`"
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh
index 722c047..f2bee2d 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/files/prepareOozieHdfsDirectories.sh
@@ -26,16 +26,17 @@ export hadoop_conf_dir=$3
export JOBTRACKER=$4
export NAMENODE=$5
export QUEUE=$6
+export JOB_NAME=$7
cd $oozie_examples_dir
/var/lib/ambari-agent/ambari-sudo.sh tar -zxf oozie-examples.tar.gz
/var/lib/ambari-agent/ambari-sudo.sh chmod -R o+rx examples
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/map-reduce/job.properties
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/map-reduce/job.properties
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|queueName=default|queueName=$QUEUE|g" examples/apps/map-reduce/job.properties
-/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/map-reduce/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:8020|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|nameNode=hdfs://localhost:9000|nameNode=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8021|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:9001|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|jobTracker=localhost:8032|jobTracker=$JOBTRACKER|g" examples/apps/$JOB_NAME/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|queueName=default|queueName=$QUEUE|g" examples/apps/$JOB_NAME/job.properties
+/var/lib/ambari-agent/ambari-sudo.sh sed -i "s|oozie.wf.application.path=hdfs://localhost:9000|oozie.wf.application.path=$NAMENODE|g" examples/apps/$JOB_NAME/job.properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index 284794d..dd4e373 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -208,6 +208,8 @@ if https_port is None and 'oozie.https.port' in config['configurations']['oozie-
oozie_base_url = config['configurations']['oozie-site']['oozie.base.url']
+service_check_job_name = default("/configurations/oozie-env/service_check_job_name", "no-op")
+
# construct proper url for https
if https_port is not None:
parsed_url = urlparse(oozie_base_url)
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
index 1a3ab08..ae7cb21 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
@@ -70,7 +70,7 @@ class OozieServiceCheckDefault(OozieServiceCheck):
raise Fail(format(NO_DOCS_FOLDER_MESSAGE))
oozie_examples_dir = oozie_examples_dir_regex_matches[0]
- Execute((format("{tmp_dir}/{prepare_hdfs_file_name}"), params.conf_dir, oozie_examples_dir, params.hadoop_conf_dir, params.yarn_resourcemanager_address, params.fs_root, params.service_check_queue_name),
+ Execute((format("{tmp_dir}/{prepare_hdfs_file_name}"), params.conf_dir, oozie_examples_dir, params.hadoop_conf_dir, params.yarn_resourcemanager_address, params.fs_root, params.service_check_queue_name, params.service_check_job_name),
tries=3,
try_sleep=5,
logoutput=True
@@ -112,10 +112,10 @@ class OozieServiceCheckDefault(OozieServiceCheck):
if params.security_enabled:
sh_cmd = format(
- "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}")
+ "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {service_check_job_name} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}")
else:
sh_cmd = format(
- "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled}")
+ "{tmp_dir}/{file_name} {os_family} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {oozie_base_url} {oozie_examples_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {service_check_job_name} {security_enabled}")
Execute(sh_cmd,
path=params.execute_path,
@@ -137,4 +137,4 @@ class OozieServiceCheckWindows(OozieServiceCheck):
if __name__ == "__main__":
OozieServiceCheck().execute()
-
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py
index c3b4ce6..468f602 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_service_check.py
@@ -40,6 +40,7 @@ class TestOozieServiceCheck(RMFTestCase):
json_content['commandParams']['version'] = version
json_content['hostLevelParams']['stack_name'] = 'HDP'
json_content['hostLevelParams']['stack_version'] = '2.2'
+ json_content['configurations']['oozie-env']['service_check_job_name'] = 'map-reduce'
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
@@ -64,7 +65,7 @@ class TestOozieServiceCheck(RMFTestCase):
mode = 0755)
self.assertResourceCalled('Execute',
- ('/tmp/prepareOozieHdfsDirectories.sh', '/usr/hdp/current/oozie-client/conf', 'examples-dir', '/usr/hdp/current/hadoop-client/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default'),
+ ('/tmp/prepareOozieHdfsDirectories.sh', '/usr/hdp/current/oozie-client/conf', 'examples-dir', '/usr/hdp/current/hadoop-client/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default', 'map-reduce'),
tries = 3,
try_sleep = 5,
logoutput = True)
http://git-wip-us.apache.org/repos/asf/ambari/blob/eca4e54e/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
index 2626578..bfa7c01 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
@@ -62,7 +62,7 @@ class TestServiceCheck(RMFTestCase):
content = StaticFile('prepareOozieHdfsDirectories.sh'),
mode = 0755,
)
- self.assertResourceCalled('Execute', ('/tmp/prepareOozieHdfsDirectories.sh', '/etc/oozie/conf', '/', '/etc/hadoop/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default'),
+ self.assertResourceCalled('Execute', ('/tmp/prepareOozieHdfsDirectories.sh', '/etc/oozie/conf', '/', '/etc/hadoop/conf', 'c6402.ambari.apache.org:8050', 'hdfs://c6401.ambari.apache.org:8020', 'default', 'no-op'),
logoutput = True,
tries = 3,
try_sleep = 5,
@@ -146,7 +146,7 @@ class TestServiceCheck(RMFTestCase):
action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
hadoop_conf_dir = '/etc/hadoop/conf',
)
- self.assertResourceCalled('Execute', '/tmp/oozieSmoke2.sh suse /var/lib/oozie /etc/oozie/conf /usr/bin http://c6402.ambari.apache.org:11000/oozie / /etc/hadoop/conf /usr/bin ambari-qa False',
+ self.assertResourceCalled('Execute', '/tmp/oozieSmoke2.sh suse /var/lib/oozie /etc/oozie/conf /usr/bin http://c6402.ambari.apache.org:11000/oozie / /etc/hadoop/conf /usr/bin ambari-qa no-op False',
logoutput = True,
path = ['/usr/bin:/usr/bin'],
tries = 3,