You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by am...@apache.org on 2017/11/16 15:37:08 UTC
[45/50] [abbrv] ambari git commit: AMBARI-22444 - Add Native
Libraries To Tez Tarball (jonathanhurley)
AMBARI-22444 - Add Native Libraries To Tez Tarball (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e9155b30
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e9155b30
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e9155b30
Branch: refs/heads/branch-feature-AMBARI-22008
Commit: e9155b309c4ff165987926f3d186ce84c0fcd19a
Parents: f122ab7
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Nov 14 16:19:30 2017 -0500
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Thu Nov 16 16:35:31 2017 +0100
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 82 +++++++++++++++++---
.../libraries/functions/tar_archive.py | 6 +-
.../stacks/2.1/FALCON/test_falcon_server.py | 6 +-
3 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e9155b30/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 219430a..f62aa3d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -23,18 +23,74 @@ __all__ = ["copy_to_hdfs", "get_sysprep_skip_copy_tarballs_hdfs"]
import os
import tempfile
import re
+import tarfile
+from contextlib import closing
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.default import default
from resource_management.core import shell
+from resource_management.core import sudo
from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
from resource_management.libraries.functions import stack_tools, stack_features, stack_select
+from resource_management.libraries.functions import tar_archive
STACK_NAME_PATTERN = "{{ stack_name }}"
STACK_ROOT_PATTERN = "{{ stack_root }}"
STACK_VERSION_PATTERN = "{{ stack_version }}"
+def _prepare_tez_tarball():
+ """
+ Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
+ :return: the full path of the newly created tez tarball to use
+ """
+ import tempfile
+
+ Logger.info("Preparing the Tez tarball...")
+
+ # get the mapreduce tarball which matches the version of tez
+ # tez installs the mapreduce tar, so it should always be present
+ _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
+ _, tez_source_file, _, _ = get_tarball_paths("tez")
+
+ temp_dir = Script.get_tmp_dir()
+
+ mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-", dir=temp_dir)
+ tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir)
+
+ Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir))
+ tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
+
+ Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
+ tar_archive.untar_archive(tez_source_file, tez_temp_dir)
+
+ hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib", "native")
+ tez_lib_dir = os.path.join(tez_temp_dir, "lib")
+
+ if not os.path.exists(hadoop_lib_native_dir):
+ raise Fail("Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist".format(hadoop_lib_native_dir))
+
+ if not os.path.exists(tez_lib_dir):
+ raise Fail("Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist".format(tez_lib_dir))
+
+ Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo = True)
+
+ tez_tarball_with_native_lib = os.path.join(os.path.dirname(tez_source_file), "tez-native.tar.gz")
+ Logger.info("Creating a new Tez tarball at {0}".format(tez_tarball_with_native_lib))
+
+ # tar up Tez, making sure to specify nothing for the arcname so that it does not include an absolute path
+ with closing(tarfile.open(tez_tarball_with_native_lib, "w:gz")) as new_tez_tarball:
+ new_tez_tarball.add(tez_temp_dir, arcname=os.path.sep)
+
+ # cleanup
+ sudo.rmtree(mapreduce_temp_dir)
+ sudo.rmtree(tez_temp_dir)
+
+ return tez_tarball_with_native_lib
+
+
# TODO, in the future, each stack can define its own mapping of tarballs
# inside the stack definition directory in some sort of xml file.
# PLEASE DO NOT put this in cluster-env since it becomes much harder to change,
@@ -50,7 +106,8 @@ TARBALL_MAP = {
"tez": {
"dirs": ("{0}/{1}/tez/lib/tez.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
"/{0}/apps/{1}/tez/tez.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
- "service": "TEZ"
+ "service": "TEZ",
+ "prepare_function": _prepare_tez_tarball
},
"tez_hive2": {
@@ -132,29 +189,29 @@ def get_tarball_paths(name, use_upgrading_version_during_upgrade=True, custom_so
:param use_upgrading_version_during_upgrade:
:param custom_source_file: If specified, use this source path instead of the default one from the map.
:param custom_dest_file: If specified, use this destination path instead of the default one from the map.
- :return: A tuple of (success status, source path, destination path)
+ :return: A tuple of (success status, source path, destination path, optional preparation function which is invoked to setup the tarball)
"""
stack_name = Script.get_stack_name()
if not stack_name:
Logger.error("Cannot copy {0} tarball to HDFS because stack name could not be determined.".format(str(name)))
- return (False, None, None)
+ return False, None, None
if name is None or name.lower() not in TARBALL_MAP:
Logger.error("Cannot copy tarball to HDFS because {0} is not supported in stack {1} for this operation.".format(str(name), str(stack_name)))
- return (False, None, None)
+ return False, None, None
service = TARBALL_MAP[name.lower()]['service']
stack_version = get_current_version(service=service, use_upgrading_version_during_upgrade=use_upgrading_version_during_upgrade)
if not stack_version:
Logger.error("Cannot copy {0} tarball to HDFS because stack version could be be determined.".format(str(name)))
- return (False, None, None)
+ return False, None, None
stack_root = Script.get_stack_root()
if not stack_root:
Logger.error("Cannot copy {0} tarball to HDFS because stack root could be be determined.".format(str(name)))
- return (False, None, None)
+ return False, None, None
(source_file, dest_file) = TARBALL_MAP[name.lower()]['dirs']
@@ -173,7 +230,11 @@ def get_tarball_paths(name, use_upgrading_version_during_upgrade=True, custom_so
source_file = source_file.replace(STACK_VERSION_PATTERN, stack_version)
dest_file = dest_file.replace(STACK_VERSION_PATTERN, stack_version)
- return (True, source_file, dest_file)
+ prepare_function = None
+ if "prepare_function" in TARBALL_MAP[name.lower()]:
+ prepare_function = TARBALL_MAP[name.lower()]['prepare_function']
+
+ return True, source_file, dest_file, prepare_function
def get_current_version(service=None, use_upgrading_version_during_upgrade=True):
@@ -272,8 +333,8 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
import params
Logger.info("Called copy_to_hdfs tarball: {0}".format(name))
- (success, source_file, dest_file) = get_tarball_paths(name, use_upgrading_version_during_upgrade,
- custom_source_file, custom_dest_file)
+ (success, source_file, dest_file, prepare_function) = get_tarball_paths(name, use_upgrading_version_during_upgrade,
+ custom_source_file, custom_dest_file)
if not success:
Logger.error("Could not copy tarball {0} due to a missing or incorrect parameter.".format(str(name)))
@@ -311,6 +372,9 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
# The logic above cannot be used until fast-hdfs-resource.jar supports the mv command, or it switches
# to WebHDFS.
+ # if there is a function which is needed to prepare the tarball, then invoke it first
+ if prepare_function is not None:
+ source_file = prepare_function()
# If the directory already exists, it is a NO-OP
dest_dir = os.path.dirname(dest_file)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e9155b30/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
index e6d8924..7976587 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -46,11 +46,13 @@ def archive_directory_dereference(archive, directory):
try_sleep = 1,
)
-def untar_archive(archive, directory):
+def untar_archive(archive, directory, silent=True):
"""
:param directory: can be a symlink and is followed
"""
- Execute(('tar','-xvf',archive,'-C',directory+"/"),
+ options = "-xf" if silent else "-xvf"
+
+ Execute(('tar',options,archive,'-C',directory+"/"),
sudo = True,
tries = 3,
try_sleep = 1,
http://git-wip-us.apache.org/repos/asf/ambari/blob/e9155b30/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 8c48347..e15cfdb 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -248,7 +248,7 @@ class TestFalconServer(RMFTestCase):
sudo = True,
)
self.assertResourceCalled('Execute', ('tar',
- '-xvf',
+ '-xf',
'/tmp/falcon-upgrade-backup/falcon-local-backup.tar',
'-C',
u'/hadoop/falcon/'),
@@ -433,7 +433,7 @@ class TestFalconServer(RMFTestCase):
self.assertResourceCalled('Execute',
('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'falcon-server', version), sudo=True,)
self.assertResourceCalled('Execute', ('tar',
- '-xvf',
+ '-xf',
'/tmp/falcon-upgrade-backup/falcon-local-backup.tar',
'-C',
u'/hadoop/falcon/'),
@@ -473,7 +473,7 @@ class TestFalconServer(RMFTestCase):
('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'falcon-server', version), sudo=True,)
self.assertResourceCalled('Execute', ('tar',
- '-xvf',
+ '-xf',
'/tmp/falcon-upgrade-backup/falcon-local-backup.tar',
'-C',
u'/hadoop/falcon/'),