You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/09/18 16:43:54 UTC
[05/34] git commit: AMBARI-7333. Tez deployment changes for Champlain
(aonishuk)
AMBARI-7333. Tez deployment changes for Champlain (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c09edb72
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c09edb72
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c09edb72
Branch: refs/heads/branch-alerts-dev
Commit: c09edb729fce602f0943b048df7810353ecd4975
Parents: 4bcf111
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Sep 17 18:11:06 2014 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Sep 17 18:11:06 2014 +0300
----------------------------------------------------------------------
.../libraries/providers/copy_from_local.py | 12 +++++++++---
.../libraries/resources/copy_from_local.py | 3 ++-
.../HIVE/package/scripts/install_jars.py | 19 +++++++++++++++----
.../2.1/services/TEZ/configuration/tez-site.xml | 2 +-
.../python/stacks/2.0.6/HIVE/test_hive_server.py | 10 ++++++++++
5 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
index 9031a77..79d3e35 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -27,6 +27,7 @@ class CopyFromLocalProvider(Provider):
def action_run(self):
path = self.resource.path
dest_dir = self.resource.dest_dir
+ dest_file = self.resource.dest_file
kinnit_if_needed = self.resource.kinnit_if_needed
owner = self.resource.owner
group = self.resource.group
@@ -34,9 +35,14 @@ class CopyFromLocalProvider(Provider):
hdfs_usr=self.resource.hdfs_user
hadoop_conf_path = self.resource.hadoop_conf_dir
- copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
- dest_file_name = os.path.split(path)[1]
- dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name
+
+ if dest_file:
+ copy_cmd = format("fs -copyFromLocal {path} {dest_dir}/{dest_file}")
+ dest_path = dest_dir + dest_file if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file
+ else:
+ dest_file_name = os.path.split(path)[1]
+ copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
+ dest_path = dest_dir + os.sep + dest_file_name
# Need to run unless as resource user
su_cmd = 'su - {0} -c'.format(owner)
unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1")
http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
index 328d9c2..eaaeab5 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
@@ -28,6 +28,7 @@ class CopyFromLocal(Resource):
path = ResourceArgument(default=lambda obj: obj.name)
dest_dir = ResourceArgument(required=True)
+ dest_file = ResourceArgument()
owner = ResourceArgument(required=True)
group = ResourceArgument()
mode = ResourceArgument()
@@ -35,4 +36,4 @@ class CopyFromLocal(Resource):
hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
hdfs_user = ResourceArgument(default='hdfs')
- actions = Resource.actions + ["run"]
\ No newline at end of file
+ actions = Resource.actions + ["run"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
index 3548de7..0045fee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
@@ -72,6 +72,16 @@ def install_tez_jars():
hdfs_user=params.hdfs_user,
hadoop_conf_dir=params.hadoop_conf_dir
)
+
+ CopyFromLocal(params.tez_tar_file,
+ owner=params.tez_user,
+ mode=0755,
+ dest_dir=app_dir_path,
+ dest_file="tez.tar.gz",
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user,
+ hadoop_conf_dir=params.hadoop_conf_dir
+ )
pass
if lib_dir_path:
@@ -92,10 +102,11 @@ def get_tez_hdfs_dir_paths(tez_lib_uris = None):
if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
dir_paths = tez_lib_uris.split(',')
for path in dir_paths:
- lib_dir_path = path.replace(hdfs_path_prefix, '')
- lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
- lib_dir_paths.append(lib_dir_path)
+ if not "tez.tar.gz" in path:
+ lib_dir_path = path.replace(hdfs_path_prefix, '')
+ lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
+ lib_dir_paths.append(lib_dir_path)
pass
pass
- return lib_dir_paths
\ No newline at end of file
+ return lib_dir_paths
http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
index 218c508..7b34d3b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
@@ -21,7 +21,7 @@
<property>
<name>tez.lib.uris</name>
- <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/</value>
+ <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/,hdfs:///apps/tez/tez.tar.gz</value>
<description>The location of the Tez libraries which will be localized for DAGs</description>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index bc723ab..e0dc3d8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -88,6 +88,16 @@ class TestHiveServer(RMFTestCase):
hdfs_user='hdfs'
)
+ self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.tar.gz',
+ mode=0755,
+ owner='tez',
+ dest_dir='/apps/tez/',
+ dest_file='tez.tar.gz',
+ kinnit_if_needed='',
+ hadoop_conf_dir='/etc/hadoop/conf',
+ hdfs_user='hdfs'
+ )
+
self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
mode=0755,
owner='tez',