You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/10/20 18:59:21 UTC
ambari git commit: AMBARI-13383. Ambari to install/manage Slider
tarball to HDFS (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 1cd72bddd -> f1e9e3460
AMBARI-13383. Ambari to install/manage Slider tarball to HDFS (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f1e9e346
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f1e9e346
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f1e9e346
Branch: refs/heads/trunk
Commit: f1e9e3460ea4c50154a91c0aaa8c827bc39f3fde
Parents: 1cd72bd
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Oct 20 19:58:53 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Oct 20 19:58:53 2015 +0300
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 2 +
.../0.60.0.2.2/package/scripts/params_linux.py | 50 ++++++++++++++++++--
.../0.60.0.2.2/package/scripts/service_check.py | 5 ++
.../SLIDER/0.60.0.2.2/package/scripts/slider.py | 5 ++
.../2.1.0.2.0/package/scripts/historyserver.py | 6 +++
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 16 +++----
.../stacks/2.0.6/YARN/test_historyserver.py | 6 ++-
.../stacks/2.2/SLIDER/test_slider_client.py | 8 ++++
8 files changed, 85 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index badf2fe..51a6a3c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -35,6 +35,8 @@ STACK_VERSION_PATTERN = "{{ stack_version }}"
TARBALL_MAP = {
"HDP": {
+ "slider": ("/usr/hdp/{0}/slider/slider.tar.gz".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/slider/slider.tar.gz".format(STACK_VERSION_PATTERN)),
"tez": ("/usr/hdp/{0}/tez/lib/tez.tar.gz".format(STACK_VERSION_PATTERN),
"/hdp/apps/{0}/tez/tez.tar.gz".format(STACK_VERSION_PATTERN)),
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py
index 5d60e21..d7bd296 100644
--- a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_linux.py
@@ -16,16 +16,60 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+from resource_management.libraries.resources import HdfsResource
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
# server configurations
config = Script.get_config()
+slider_home_dir = '/usr/hdp/current/slider-client'
+
#hadoop params
slider_bin_dir = "/usr/lib/slider/bin"
if Script.is_hdp_stack_greater_or_equal("2.2"):
- slider_bin_dir = '/usr/hdp/current/slider-client/bin'
+ slider_bin_dir = format('{slider_home_dir}/bin')
-slider_conf_dir = "/usr/hdp/current/slider-client/conf"
+slider_conf_dir = format("{slider_home_dir}/conf")
storm_slider_conf_dir = '/usr/hdp/current/storm-slider-client/conf'
-slider_home_dir = '/usr/hdp/current/slider-client'
+
+slider_tar_gz = format('{slider_home_dir}/slider.tar.gz')
+slider_lib_dir = format('{slider_home_dir}/lib')
+
+user_group = config['configurations']['cluster-env']['user_group']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+
+hadoop_bin_dir = hdp_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/service_check.py
index 18711d7..1aed032 100644
--- a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/service_check.py
@@ -19,6 +19,7 @@ limitations under the License.
"""
from resource_management import *
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -36,6 +37,10 @@ class SliderServiceCheck(Script):
def service_check(self, env):
import params
env.set_params(params)
+
+ if Script.is_hdp_stack_greater_or_equal("2.2"):
+ copy_to_hdfs("slider", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
+
smokeuser_kinit_cmd = format(
"{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal};") if params.security_enabled else ""
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider.py b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider.py
index a802c19..1ede7f9 100644
--- a/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider.py
+++ b/ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider.py
@@ -80,3 +80,8 @@ def slider():
File(format("{params.slider_conf_dir}/log4j.properties"),
mode=0644
)
+ if Script.is_hdp_stack_greater_or_equal("2.2"):
+ Execute(('tar', '-czvf', params.slider_tar_gz, '--exclude=slider-core*.jar', params.slider_lib_dir),
+ not_if = format('test -f {slider_tar_gz}'),
+ sudo = True,
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
index 5fe8157..229dcd8 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
@@ -83,6 +83,7 @@ class HistoryServerDefault(HistoryServer):
# MC Hammer said, "Can't touch this"
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
+ copy_to_hdfs("slider", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
params.HdfsResource(None, action="execute")
def start(self, env, rolling_restart=False):
@@ -102,6 +103,11 @@ class HistoryServerDefault(HistoryServer):
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped) or resource_created
+ resource_created = copy_to_hdfs(
+ "slider",
+ params.user_group,
+ params.hdfs_user,
+ host_sys_prepped=params.host_sys_prepped) or resource_created
if resource_created:
params.HdfsResource(None, action="execute")
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index e61bd67..5180afa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -439,6 +439,14 @@
</upgrade>
</component>
</service>
+
+ <service name="SLIDER">
+ <component name="SLIDER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
<service name="MAPREDUCE2">
<component name="HISTORYSERVER">
@@ -588,14 +596,6 @@
</component>
</service>
- <service name="SLIDER">
- <component name="SLIDER">
- <upgrade>
- <task xsi:type="restart-task"/>
- </upgrade>
- </component>
- </service>
-
<service name="SPARK">
<component name="SPARK_JOBHISTORYSERVER">
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
index 85643bc..1321aaa 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
@@ -18,7 +18,7 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
import json
-from mock.mock import MagicMock, patch
+from mock.mock import MagicMock, patch, call
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions import version
from stacks.utils.RMFTestCase import *
@@ -786,7 +786,9 @@ class TestHistoryServer(RMFTestCase):
mocks_dict = mocks_dict)
self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-mapreduce-historyserver', version), sudo=True)
- copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs", host_sys_prepped=False)
+ self.assertTrue(call("tez", "hadoop", "hdfs", host_sys_prepped=False) in copy_to_hdfs_mock.call_args_list)
+ self.assertTrue(call("slider", "hadoop", "hdfs", host_sys_prepped=False) in copy_to_hdfs_mock.call_args_list)
+
self.assertResourceCalled('HdfsResource', None,
security_enabled = False,
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1e9e346/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py b/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
index f83cf7b..9329543 100644
--- a/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
+++ b/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
@@ -66,6 +66,14 @@ class TestSliderClient(RMFTestCase):
mode=0644,
content='log4jproperties\nline2'
)
+ self.assertResourceCalled('Execute', ('tar',
+ '-czvf',
+ '/usr/hdp/current/slider-client/slider.tar.gz',
+ '--exclude=slider-core*.jar',
+ '/usr/hdp/current/slider-client/lib'),
+ not_if = 'test -f /usr/hdp/current/slider-client/slider.tar.gz',
+ sudo = True,
+ )
self.assertNoMoreResources()