You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2014/02/27 22:58:06 UTC
git commit: AMBARI-4862. Hive execute JAR needs to be installed on
HDFS. (Siddharth Wagle via mahadev)
Repository: ambari
Updated Branches:
refs/heads/trunk 0ee9b81f9 -> 9acbc1001
AMBARI-4862. Hive execute JAR needs to be installed on HDFS. (Siddharth Wagle via mahadev)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9acbc100
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9acbc100
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9acbc100
Branch: refs/heads/trunk
Commit: 9acbc10014bf2e752598cf3ca94499a30b7b96dd
Parents: 0ee9b81
Author: Mahadev Konar <ma...@apache.org>
Authored: Thu Feb 27 13:57:59 2014 -0800
Committer: Mahadev Konar <ma...@apache.org>
Committed: Thu Feb 27 13:57:59 2014 -0800
----------------------------------------------------------------------
.../libraries/providers/copy_from_local.py | 11 +++---
.../libraries/resources/copy_from_local.py | 5 ++-
.../resource_management/TestCopyFromLocal.py | 14 +++----
.../HIVE/package/scripts/hive_server.py | 41 +++++++++++++++++++-
.../services/HIVE/package/scripts/params.py | 4 +-
.../services/WEBHCAT/package/scripts/webhcat.py | 9 +++--
.../stacks/2.0.6/HIVE/test_hive_server.py | 26 ++++++++++++-
.../stacks/2.0.6/WEBHCAT/test_webhcat_server.py | 18 ++++++---
8 files changed, 100 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
index ef47a87..6d04e72 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -27,10 +27,10 @@ class CopyFromLocalProvider(Provider):
path = self.resource.path
dest_dir = self.resource.dest_dir
kinnit_if_needed = self.resource.kinnit_if_needed
- stub_path = self.resource.stub_path
owner = self.resource.owner
group = self.resource.group
mode = self.resource.mode
+ hdfs_usr=self.resource.hdfs_user
hadoop_conf_path = self.resource.hadoop_conf_dir
copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
@@ -39,7 +39,7 @@ class CopyFromLocalProvider(Provider):
ExecuteHadoop(copy_cmd,
not_if=unless_cmd,
user=owner,
- conf_dir=hadoop_conf_path,
+ conf_dir=hadoop_conf_path
)
if not owner:
@@ -54,14 +54,15 @@ class CopyFromLocalProvider(Provider):
chown_cmd = format("fs -chown {chown} {dest_dir}")
ExecuteHadoop(chown_cmd,
- user=owner,
+ user=hdfs_usr,
conf_dir=hadoop_conf_path)
pass
if mode:
- chmod_cmd = format('fs -chmod {mode} {dest_dir}')
+ dir_mode = oct(mode)
+ chmod_cmd = format('fs -chmod {dir_mode} {dest_dir}')
ExecuteHadoop(chmod_cmd,
- user=owner,
+ user=hdfs_usr,
conf_dir=hadoop_conf_path)
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py b/ambari-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py
index 829d6b8..328d9c2 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py
@@ -27,11 +27,12 @@ class CopyFromLocal(Resource):
action = ForcedListArgument(default="run")
path = ResourceArgument(default=lambda obj: obj.name)
- dest_dir = ResourceArgument()
- owner = ResourceArgument()
+ dest_dir = ResourceArgument(required=True)
+ owner = ResourceArgument(required=True)
group = ResourceArgument()
mode = ResourceArgument()
kinnit_if_needed = ResourceArgument(default='')
hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
+ hdfs_user = ResourceArgument(default='hdfs')
actions = Resource.actions + ["run"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
index d65f904..40b7fc8 100644
--- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
+++ b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
@@ -30,17 +30,16 @@ class TestCopyFromLocal(TestCase):
owner='user1',
dest_dir='/apps/test/',
kinnit_if_needed='',
- stub_path='/tmp/test_stub',
- ignore_on_failure=True
+ hdfs_user='hdfs'
)
self.assertEqual(execute_hadoop_mock.call_count, 2)
call_arg_list = execute_hadoop_mock.call_args_list
self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': ' hadoop fs -ls /tmp/test_stub >/dev/null 2>&1', 'ignore_failures': False, 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+ self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/ >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
call_arg_list[0][0][0].arguments)
self.assertEquals('fs -chown user1 /apps/test/', call_arg_list[1][0][0].command)
- self.assertEquals({'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
+ self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
@patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
@@ -52,16 +51,15 @@ class TestCopyFromLocal(TestCase):
group='hdfs',
dest_dir='/apps/test/',
kinnit_if_needed='',
- stub_path='/tmp/test_stub',
- ignore_on_failure=False
+ hdfs_user='hdfs'
)
self.assertEqual(execute_hadoop_mock.call_count, 3)
call_arg_list = execute_hadoop_mock.call_args_list
self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': ' hadoop fs -ls /tmp/test_stub >/dev/null 2>&1', 'ignore_failures': False, 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+ self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/ >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
call_arg_list[0][0][0].arguments)
self.assertEquals('fs -chown user1:hdfs /apps/test/', call_arg_list[1][0][0].command)
- self.assertEquals({'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
+ self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
index dd247dd..c883cc1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
@@ -40,6 +40,7 @@ class HiveServer(Script):
env.set_params(params)
self.configure(env) # FOR SECURITY
self.install_tez_jars(params) # Put tez jars in hdfs
+ self.install_hive_exec_jar(params) # Put hive exec jar in hdfs
hive_service( 'hiveserver2',
action = 'start'
)
@@ -59,6 +60,41 @@ class HiveServer(Script):
# Recursively check all existing gmetad pid files
check_process_status(pid_file)
+ def install_hive_exec_jar(self, params):
+ hdfs_path_prefix = 'hdfs://'
+ if params.tez_lib_uris:
+ hdfs_path = params.hive_exec_hdfs_path
+
+ if hdfs_path.strip().find(hdfs_path_prefix, 0) != -1:
+ hdfs_path = hdfs_path.replace(hdfs_path_prefix, '')
+ pass
+
+ params.HdfsDirectory(hdfs_path,
+ action="create",
+ owner=params.hive_user,
+ mode=0755
+ )
+
+ if params.security_enabled:
+ kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};")
+ else:
+ kinit_if_needed = ""
+
+ if kinit_if_needed:
+ Execute(kinit_if_needed,
+ user=params.tez_user,
+ path='/bin'
+ )
+
+ CopyFromLocal(params.hive_exec_jar_path,
+ mode=0655,
+ owner=params.hive_user,
+ dest_dir=hdfs_path,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
+ )
+ pass
+
def install_tez_jars(self, params):
destination_hdfs_dirs = get_tez_hdfs_dir_paths(params.tez_lib_uris)
@@ -83,6 +119,7 @@ class HiveServer(Script):
user=params.tez_user,
path='/bin'
)
+ pass
app_dir_path = None
lib_dir_path = None
@@ -102,7 +139,8 @@ class HiveServer(Script):
mode=0655,
owner=params.tez_user,
dest_dir=app_dir_path,
- kinnit_if_needed=kinit_if_needed
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
pass
@@ -112,6 +150,7 @@ class HiveServer(Script):
owner=params.tez_user,
dest_dir=lib_dir_path,
kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
index 24998d8..9961212 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
@@ -155,9 +155,11 @@ kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "
tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None)
tez_local_api_jars = '/usr/lib/tez/tez*.jar'
tez_local_lib_jars = '/usr/lib/tez/lib/*.jar'
-tez_stub_path = '/tmp/tez_jars_copied'
tez_user = 'tez'
+hive_exec_jar_path = '/usr/lib/hive/lib/hive-exec.jar'
+hive_exec_hdfs_path = default('/configurations/hive-site/hive.jar.directory', '/apps/hive/install')
+
import functools
#create partial functions with common arguments for every HdfsDirectory call
#to create hdfs directory we need to call params.HdfsDirectory in code
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
index 726fc76..67bb1a6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
@@ -88,19 +88,22 @@ def webhcat():
owner=params.webhcat_user,
mode=0755,
dest_dir=format("{webhcat_apps_dir}/hadoop-streaming.jar"),
- kinnit_if_needed=kinit_if_needed
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz',
owner=params.webhcat_user,
mode=0755,
dest_dir=format("{webhcat_apps_dir}/pig.tar.gz"),
- kinnit_if_needed=kinit_if_needed
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz',
owner=params.webhcat_user,
mode=0755,
dest_dir=format("{webhcat_apps_dir}/hive.tar.gz"),
- kinnit_if_needed=kinit_if_needed
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 4877c2e..10f133d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -102,16 +102,38 @@ class TestHiveServer(RMFTestCase):
mode=0655,
owner='tez',
dest_dir='/apps/tez/',
- kinnit_if_needed=''
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
mode=0655,
owner='tez',
dest_dir='/apps/tez/lib/',
- kinnit_if_needed=''
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
+ self.assertResourceCalled('HdfsDirectory', '/apps/hive/install',
+ security_enabled = False,
+ mode = 0755,
+ owner = 'hive',
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ kinit_path_local = '/usr/bin/kinit',
+ action = ['create']
+ )
+
+ self.assertResourceCalled('CopyFromLocal', '/usr/lib/hive/lib/hive-exec.jar',
+ mode=0655,
+ owner='hive',
+ dest_dir='/apps/hive/install',
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
+ )
+
+
self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server',
not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1',
user = 'hive'
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acbc100/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
index 2b5e3ae..7fb7b57 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
@@ -158,19 +158,22 @@ class TestWebHCatServer(RMFTestCase):
owner='hcat',
mode=0755,
dest_dir=format('/apps/webhcat/hadoop-streaming.jar'),
- kinnit_if_needed=''
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
owner='hcat',
mode=0755,
dest_dir=format('/apps/webhcat/pig.tar.gz'),
- kinnit_if_needed=''
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
owner='hcat',
mode=0755,
dest_dir=format('/apps/webhcat/hive.tar.gz'),
- kinnit_if_needed=''
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
def assert_configure_secured(self):
@@ -237,17 +240,20 @@ class TestWebHCatServer(RMFTestCase):
owner='hcat',
mode=0755,
dest_dir=format('/apps/webhcat/hadoop-streaming.jar'),
- kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;'
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+ hdfs_user='hdfs'
)
self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
owner='hcat',
mode=0755,
dest_dir=format('/apps/webhcat/pig.tar.gz'),
- kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;'
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+ hdfs_user='hdfs'
)
self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
owner='hcat',
mode=0755,
dest_dir=format('/apps/webhcat/hive.tar.gz'),
- kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;'
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+ hdfs_user='hdfs'
)
\ No newline at end of file