You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/04/11 20:48:08 UTC
git commit: AMBARI-5447. WebHCat start Fail with authorization
enabled. (swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-1.5.1 583f2410c -> 230dbf8f7
AMBARI-5447. WebHCat start Fail with authorization enabled. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/230dbf8f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/230dbf8f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/230dbf8f
Branch: refs/heads/branch-1.5.1
Commit: 230dbf8f79344f57d00ab1c0d509af3a98bda0d1
Parents: 583f241
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Fri Apr 11 11:47:28 2014 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Fri Apr 11 11:47:28 2014 -0700
----------------------------------------------------------------------
.../libraries/providers/copy_from_local.py | 5 +-
.../resource_management/TestCopyFromLocal.py | 4 +-
.../services/WEBHCAT/package/scripts/webhcat.py | 54 ++++-------------
.../stacks/1.3.2/WEBHCAT/test_webhcat_server.py | 62 ++++++++++++--------
4 files changed, 54 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
index d25dd8b..3fa8a54 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -37,8 +37,9 @@ class CopyFromLocalProvider(Provider):
copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
dest_file_name = os.path.split(path)[1]
dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name
-
- unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_path} >/dev/null 2>&1")
+ # Need to run unless as resource user
+ su_cmd = 'su - {0} -c'.format(owner)
+ unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1")
ExecuteHadoop(copy_cmd,
not_if=unless_cmd,
http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
index 7653b24..90397ae 100644
--- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
+++ b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
@@ -36,7 +36,7 @@ class TestCopyFromLocal(TestCase):
call_arg_list = execute_hadoop_mock.call_args_list
self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+ self.assertEquals({'not_if': "su - user1 -c ' hadoop fs -ls /apps/test/*.files' >/dev/null 2>&1", 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
call_arg_list[0][0][0].arguments)
self.assertEquals('fs -chown user1 /apps/test/*.files', call_arg_list[1][0][0].command)
self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
@@ -57,7 +57,7 @@ class TestCopyFromLocal(TestCase):
call_arg_list = execute_hadoop_mock.call_args_list
self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+ self.assertEquals({'not_if': "su - user1 -c ' hadoop fs -ls /apps/test/*.files' >/dev/null 2>&1", 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
call_arg_list[0][0][0].arguments)
self.assertEquals('fs -chown user1:hdfs /apps/test/*.files', call_arg_list[1][0][0].command)
self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
index 0c75d18..be592d7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
@@ -20,7 +20,6 @@ Ambari Agent
"""
from resource_management import *
-import sys
def webhcat():
@@ -83,55 +82,26 @@ def webhcat():
path='/bin'
)
- copyFromLocal(path='/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
+ CopyFromLocal('/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
owner=params.webhcat_user,
mode=0755,
- dest_dir=format("{webhcat_apps_dir}/hadoop-streaming.jar"),
- kinnit_if_needed=kinit_if_needed
+ dest_dir=params.webhcat_apps_dir,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
- copyFromLocal(path='/usr/share/HDP-webhcat/pig.tar.gz',
+ CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz',
owner=params.webhcat_user,
mode=0755,
- dest_dir=format("{webhcat_apps_dir}/pig.tar.gz"),
+ dest_dir=params.webhcat_apps_dir,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
- copyFromLocal(path='/usr/share/HDP-webhcat/hive.tar.gz',
+ CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz',
owner=params.webhcat_user,
mode=0755,
- dest_dir=format("{webhcat_apps_dir}/hive.tar.gz")
+ dest_dir=params.webhcat_apps_dir,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user
)
-
-
-def copyFromLocal(path=None, owner=None, group=None, mode=None, dest_dir=None, kinnit_if_needed=""):
- import params
-
- copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
- unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_dir} >/dev/null 2>&1")
-
- ExecuteHadoop(copy_cmd,
- not_if=unless_cmd,
- user=owner,
- conf_dir=params.hadoop_conf_dir)
-
- if not owner:
- chown = None
- else:
- if not group:
- chown = owner
- else:
- chown = format('{owner}:{group}')
-
- if not chown:
- chown_cmd = format("fs -chown {chown} {dest_dir}")
-
- ExecuteHadoop(copy_cmd,
- user=owner,
- conf_dir=params.hadoop_conf_dir)
-
- if not mode:
- chmod_cmd = format('fs -chmod {mode} {dest_dir}')
-
- ExecuteHadoop(chmod_cmd,
- user=owner,
- conf_dir=params.hadoop_conf_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py b/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py
index 666d693..8b363fd 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py
@@ -151,20 +151,26 @@ class TestWebHCatServer(RMFTestCase):
owner = 'hcat',
group = 'hadoop',
)
- self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar',
- not_if = ' hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
- user = 'hcat',
- conf_dir = '/etc/hadoop/conf',
+ self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
- self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz',
- not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1',
- user = 'hcat',
- conf_dir = '/etc/hadoop/conf',
+ self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
- self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz',
- not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1',
- user = 'hcat',
- conf_dir = '/etc/hadoop/conf',
+ self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='',
+ hdfs_user='hdfs'
)
def assert_configure_secured(self):
@@ -227,18 +233,24 @@ class TestWebHCatServer(RMFTestCase):
path = ['/bin'],
user = 'hcat',
)
- self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar',
- not_if = '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
- user = 'hcat',
- conf_dir = '/etc/hadoop/conf',
+ self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+ hdfs_user='hdfs'
)
- self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz',
- not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1',
- user = 'hcat',
- conf_dir = '/etc/hadoop/conf',
- )
- self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz',
- not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1',
- user = 'hcat',
- conf_dir = '/etc/hadoop/conf',
+ self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+ hdfs_user='hdfs'
)
+ self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+ hdfs_user='hdfs'
+ )
\ No newline at end of file