You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/11/28 23:29:20 UTC
ambari git commit: AMBARI-8476. Run sudo commands as sudo and sudo su
ability in Resource Management (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 0fc4e7f0e -> 5d07646ee
AMBARI-8476. Run sudo commands as sudo and sudo su ability in Resource Management (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5d07646e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5d07646e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5d07646e
Branch: refs/heads/trunk
Commit: 5d07646eedc1965def6c1b81d9bcea8b5ddc5673
Parents: 0fc4e7f
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Sat Nov 29 00:29:08 2014 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Sat Nov 29 00:29:08 2014 +0200
----------------------------------------------------------------------
.../resource_management/TestCopyFromLocal.py | 8 +-
.../resource_management/TestExecuteResource.py | 4 +-
.../resource_management/TestGroupResource.py | 10 +--
.../resource_management/TestPackageResource.py | 24 +++---
.../resource_management/TestUserResource.py | 22 ++---
.../core/providers/package/apt.py | 22 ++---
.../core/providers/package/yumrpm.py | 13 +--
.../core/providers/package/zypper.py | 15 ++--
.../core/providers/system.py | 1 +
.../core/resources/system.py | 15 +++-
.../python/resource_management/core/shell.py | 86 ++++++++++++++++----
.../python/resource_management/core/sudo.py | 2 +-
.../libraries/providers/copy_from_local.py | 5 +-
13 files changed, 149 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
index 32a7963..3864a7c 100644
--- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
+++ b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
@@ -17,9 +17,11 @@ limitations under the License.
'''
from unittest import TestCase
-from mock.mock import patch
+from mock.mock import patch, MagicMock
from resource_management import *
+from resource_management.core import shell
+@patch.object(shell, "call", new = MagicMock(return_value=(1, "")))
@patch.object(System, "os_family", new = 'redhat')
class TestCopyFromLocal(TestCase):
@@ -36,7 +38,7 @@ class TestCopyFromLocal(TestCase):
call_arg_list = execute_hadoop_mock.call_args_list
self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': "su - user1 -c ' export PATH=$PATH:/usr/bin ; hadoop fs -ls /apps/test//*.files' >/dev/null 2>&1", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'},
+ self.assertEquals({'not_if': "/usr/bin/sudo -Hi su - user1 -s /bin/bash -c 'export PATH=/usr/bin ; {kinnit_if_needed} ; hadoop fs -ls {dest_path}'", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'},
call_arg_list[0][0][0].arguments)
self.assertEquals('fs -chown user1 /apps/test//*.files', call_arg_list[1][0][0].command)
self.assertEquals({'user': 'hdfs', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
@@ -57,7 +59,7 @@ class TestCopyFromLocal(TestCase):
call_arg_list = execute_hadoop_mock.call_args_list
self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
call_arg_list[0][0][0].command)
- self.assertEquals({'not_if': "su - user1 -c ' export PATH=$PATH:/usr/bin ; hadoop fs -ls /apps/test//*.files' >/dev/null 2>&1", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'},
+ self.assertEquals({'not_if': "/usr/bin/sudo -Hi su - user1 -s /bin/bash -c 'export PATH=/usr/bin ; {kinnit_if_needed} ; hadoop fs -ls {dest_path}'", 'user': 'user1', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'},
call_arg_list[0][0][0].arguments)
self.assertEquals('fs -chown user1:hdfs /apps/test//*.files', call_arg_list[1][0][0].command)
self.assertEquals({'user': 'hdfs', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
index 7820e47..b2af4db 100644
--- a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
@@ -176,8 +176,8 @@ class TestExecuteResource(TestCase):
environment={'JAVA_HOME': '/test/java/home',
'PATH': "/bin"}
)
- expected_command = '/usr/bin/sudo -Hsu test_user <<< \'export PATH=' + os.environ['PATH'] + ':/bin JAVA_HOME=/test/java/home; echo "1"\''
- self.assertEqual(popen_mock.call_args_list[0][0][0][3], expected_command)
+ expected_command = ['/usr/bin/sudo', '-Hi', 'su', '-', 'test_user', '-s', '/bin/bash', '-c', 'export PATH=' + os.environ['PATH'] + ':/bin JAVA_HOME=/test/java/home ; echo "1"']
+ self.assertEqual(popen_mock.call_args_list[0][0][0], expected_command)
@patch.object(subprocess, "Popen")
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestGroupResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestGroupResource.py b/ambari-agent/src/test/python/resource_management/TestGroupResource.py
index 4245fdf..f80f78a 100644
--- a/ambari-agent/src/test/python/resource_management/TestGroupResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestGroupResource.py
@@ -45,7 +45,7 @@ class TestGroupResource(TestCase):
self.assertEqual(popen_mock.call_count, 1)
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'groupadd -p secure hadoop'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi groupadd -p secure hadoop"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
getgrnam_mock.assert_called_with('hadoop')
@@ -66,7 +66,7 @@ class TestGroupResource(TestCase):
self.assertEqual(popen_mock.call_count, 1)
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'groupmod -p secure -g 2 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
getgrnam_mock.assert_called_with('mapred')
@@ -90,7 +90,7 @@ class TestGroupResource(TestCase):
except Fail:
pass
self.assertEqual(popen_mock.call_count, 1)
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'groupmod -p secure -g 2 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
getgrnam_mock.assert_called_with('mapred')
@@ -110,7 +110,7 @@ class TestGroupResource(TestCase):
self.assertEqual(popen_mock.call_count, 1)
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
getgrnam_mock.assert_called_with('mapred')
@@ -134,5 +134,5 @@ class TestGroupResource(TestCase):
pass
self.assertEqual(popen_mock.call_count, 1)
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'groupdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
getgrnam_mock.assert_called_with('mapred')
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestPackageResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestPackageResource.py b/ambari-agent/src/test/python/resource_management/TestPackageResource.py
index c762355..95e053f 100644
--- a/ambari-agent/src/test/python/resource_management/TestPackageResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestPackageResource.py
@@ -36,13 +36,13 @@ class TestPackageResource(TestCase):
Package("some_package",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"),
- call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'"
- " --allow-unauthenticated --assume-yes install some-package", sudo=True),
- call("apt-get update -qq", sudo=True)
+ call(['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::='--force-confdef'", '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'],
+ sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'}),
+ call(['/usr/bin/apt-get', 'update', '-qq'], sudo=True)
])
- shell_mock.assert_has_calls([call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef' --allow-unauthenticated --assume-yes install some-package", sudo=True)
- ])
+ shell_mock.assert_has_calls([call(['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::='--force-confdef'", '--allow-unauthenticated', '--assume-yes', 'install',
+ 'some-package'], sudo=True)])
@patch.object(shell, "call")
@patch.object(shell, "checked_call")
@@ -53,8 +53,8 @@ class TestPackageResource(TestCase):
Package("some_package",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"),
- call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'"
- " --allow-unauthenticated --assume-yes install some-package", sudo=True)
+ call(['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::='--force-confdef'", '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'],
+ sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})
])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
@@ -70,7 +70,7 @@ class TestPackageResource(TestCase):
Package("some_package",
)
call_mock.assert_called_with('installed_pkgs=`rpm -qa some_package` ; [ ! -z "$installed_pkgs" ]')
- shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y install some_package", sudo=True)
+ shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package'], sudo=True)
@patch.object(shell, "call")
@patch.object(shell, "checked_call")
@@ -81,7 +81,7 @@ class TestPackageResource(TestCase):
Package("some_package",
)
call_mock.assert_called_with('rpm -qa | grep ^some_package')
- shell_mock.assert_called_with("/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm some_package", sudo=True)
+ shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm', 'some_package'], sudo=True)
@patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
@patch.object(shell, "checked_call")
@@ -109,7 +109,7 @@ class TestPackageResource(TestCase):
Package("some_package",
action = "remove"
)
- shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y erase some_package", sudo=True)
+ shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'erase', 'some_package'], sudo=True)
@patch.object(shell, "call", new = MagicMock(return_value=(0, None)))
@patch.object(shell, "checked_call")
@@ -119,7 +119,7 @@ class TestPackageResource(TestCase):
Package("some_package",
action = "remove"
)
- shell_mock.assert_called_with("/usr/bin/zypper --quiet remove --no-confirm some_package", sudo=True)
+ shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm', 'some_package'], sudo=True)
@patch.object(shell, "call", new = MagicMock(return_value=(1, None)))
@patch.object(shell, "checked_call")
@@ -129,7 +129,7 @@ class TestPackageResource(TestCase):
Package("some_package",
version = "3.5.0"
)
- shell_mock.assert_called_with("/usr/bin/yum -d 0 -e 0 -y install some_package-3.5.0", sudo=True)
+ shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package-3.5.0'], sudo=True)
@replace_underscores
def func_to_test(self, name):
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-agent/src/test/python/resource_management/TestUserResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestUserResource.py b/ambari-agent/src/test/python/resource_management/TestUserResource.py
index f1d22e6..86b23c5 100644
--- a/ambari-agent/src/test/python/resource_management/TestUserResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestUserResource.py
@@ -38,7 +38,7 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "create", shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'useradd -m -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi useradd -m -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -52,7 +52,7 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "create", shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -66,7 +66,7 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "remove", shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'userdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', 'userdel mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -81,7 +81,7 @@ class TestUserResource(TestCase):
user = User("mapred", action = "create", comment = "testComment",
shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -c testComment -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -c testComment -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -96,7 +96,7 @@ class TestUserResource(TestCase):
user = User("mapred", action = "create", home = "/test/home",
shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -d /test/home mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -d /test/home mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -111,7 +111,7 @@ class TestUserResource(TestCase):
user = User("mapred", action = "create", password = "secure",
shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -p secure mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -p secure mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -125,7 +125,7 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "create", shell = "/bin/sh")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/sh mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/sh mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -139,7 +139,7 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "create", uid = "1", shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -u 1 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -u 1 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -153,7 +153,7 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "create", gid = "1", shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -s /bin/bash -g 1 mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -s /bin/bash -g 1 mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -168,7 +168,7 @@ class TestUserResource(TestCase):
user = User("mapred", action = "create", groups = ['1','2','3'],
shell = "/bin/bash")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'usermod -G 1,2,3 -s /bin/bash mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi usermod -G 1,2,3 -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
@patch.object(subprocess, "Popen")
@@ -181,5 +181,5 @@ class TestUserResource(TestCase):
with Environment('/') as env:
user = User("mapred", action = "create")
- popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -s <<< 'useradd -m mapred'"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None)
+ popen_mock.assert_called_with(['/bin/bash', '--login', '-c', "/usr/bin/sudo -Hi useradd -m mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={}, cwd=None)
self.assertEqual(popen_mock.call_count, 1)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/package/apt.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py
index 0ad6ca5..c683be5 100644
--- a/ambari-common/src/main/python/resource_management/core/providers/package/apt.py
+++ b/ambari-common/src/main/python/resource_management/core/providers/package/apt.py
@@ -21,12 +21,14 @@ Ambari Agent
from resource_management.core.providers.package import PackageProvider
from resource_management.core import shell
+from resource_management.core.shell import string_cmd_from_args_list
from resource_management.core.logger import Logger
-INSTALL_CMD = "DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef' --allow-unauthenticated --assume-yes install %s"
-REPO_UPDATE_CMD = "apt-get update -qq"
-REMOVE_CMD = "/usr/bin/apt-get -y -q remove %s"
-CHECK_CMD = "dpkg --get-selections | grep ^%s$ | grep -v deinstall"
+INSTALL_CMD_ENV = {'DEBIAN_FRONTEND':'noninteractive'}
+INSTALL_CMD = ['/usr/bin/apt-get', '-q', '-o', "Dpkg::Options::=--force-confdef", '--allow-unauthenticated', '--assume-yes', 'install']
+REPO_UPDATE_CMD = ['/usr/bin/apt-get', 'update','-qq']
+REMOVE_CMD = ['/usr/bin/apt-get', '-y', '-q', 'remove']
+CHECK_CMD = "dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^%s$"
def replace_underscores(function_to_decorate):
def wrapper(*args):
@@ -40,14 +42,14 @@ class AptProvider(PackageProvider):
@replace_underscores
def install_package(self, name):
if not self._check_existence(name):
- cmd = INSTALL_CMD % (name)
- Logger.info("Installing package %s ('%s')" % (name, cmd))
- code, out = shell.call(cmd, sudo=True)
+ cmd = INSTALL_CMD + [name]
+ Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
+ code, out = shell.call(cmd, sudo=True, env=INSTALL_CMD_ENV)
# apt-get update wasn't done too long
if code:
Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out))
- Logger.info("Failed to install package %s. Executing `%s`" % (name, REPO_UPDATE_CMD))
+ Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD)))
code, out = shell.call(REPO_UPDATE_CMD, sudo=True)
if code:
@@ -65,8 +67,8 @@ class AptProvider(PackageProvider):
@replace_underscores
def remove_package(self, name):
if self._check_existence(name):
- cmd = REMOVE_CMD % (name)
- Logger.info("Removing package %s ('%s')" % (name, cmd))
+ cmd = REMOVE_CMD + [name]
+ Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True)
else:
Logger.info("Skipping removing non-existent package %s" % (name))
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py
index b026ab4..7d868b0 100644
--- a/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py
+++ b/ambari-common/src/main/python/resource_management/core/providers/package/yumrpm.py
@@ -22,17 +22,18 @@ Ambari Agent
from resource_management.core.providers.package import PackageProvider
from resource_management.core import shell
+from resource_management.core.shell import string_cmd_from_args_list
from resource_management.core.logger import Logger
-INSTALL_CMD = "/usr/bin/yum -d 0 -e 0 -y install %s"
-REMOVE_CMD = "/usr/bin/yum -d 0 -e 0 -y erase %s"
+INSTALL_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install']
+REMOVE_CMD = ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'erase']
CHECK_CMD = "installed_pkgs=`rpm -qa %s` ; [ ! -z \"$installed_pkgs\" ]"
class YumProvider(PackageProvider):
def install_package(self, name):
if not self._check_existence(name):
- cmd = INSTALL_CMD % (name)
- Logger.info("Installing package %s ('%s')" % (name, cmd))
+ cmd = INSTALL_CMD + [name]
+ Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True)
else:
Logger.info("Skipping installing existent package %s" % (name))
@@ -42,8 +43,8 @@ class YumProvider(PackageProvider):
def remove_package(self, name):
if self._check_existence(name):
- cmd = REMOVE_CMD % (name)
- Logger.info("Removing package %s ('%s')" % (name, cmd))
+ cmd = REMOVE_CMD + [name]
+ Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True)
else:
Logger.info("Skipping removing non-existent package %s" % (name))
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py
index 293a454..764da73 100644
--- a/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py
+++ b/ambari-common/src/main/python/resource_management/core/providers/package/zypper.py
@@ -22,17 +22,18 @@ Ambari Agent
from resource_management.core.providers.package import PackageProvider
from resource_management.core import shell
+from resource_management.core.shell import string_cmd_from_args_list
from resource_management.core.logger import Logger
-INSTALL_CMD = "/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm %s"
-REMOVE_CMD = "/usr/bin/zypper --quiet remove --no-confirm %s"
-CHECK_CMD = "rpm -qa | grep ^%s"
+INSTALL_CMD = ['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm']
+REMOVE_CMD = ['/usr/bin/zypper', '--quiet', 'remove', '--no-confirm']
+CHECK_CMD = "installed_pkgs=`rpm -qa %s` ; [ ! -z \"$installed_pkgs\" ]"
class ZypperProvider(PackageProvider):
def install_package(self, name):
if not self._check_existence(name):
- cmd = INSTALL_CMD % (name)
- Logger.info("Installing package %s ('%s')" % (name, cmd))
+ cmd = INSTALL_CMD + [name]
+ Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True)
else:
Logger.info("Skipping installing existent package %s" % (name))
@@ -42,8 +43,8 @@ class ZypperProvider(PackageProvider):
def remove_package(self, name):
if self._check_existence(name):
- cmd = REMOVE_CMD % (name)
- Logger.info("Removing package %s ('%s')" % (name, cmd))
+ cmd = REMOVE_CMD + [name]
+ Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True)
else:
Logger.info("Skipping removing non-existent package %s" % (name))
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/providers/system.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/providers/system.py b/ambari-common/src/main/python/resource_management/core/providers/system.py
index 6ec5bb5..db58898 100644
--- a/ambari-common/src/main/python/resource_management/core/providers/system.py
+++ b/ambari-common/src/main/python/resource_management/core/providers/system.py
@@ -246,6 +246,7 @@ class ExecuteProvider(Provider):
wait_for_finish=self.resource.wait_for_finish,
timeout=self.resource.timeout,
path=self.resource.path,
+ output_file=self.resource.output_file,
sudo=self.resource.sudo)
break
except Fail as ex:
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/resources/system.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/resources/system.py b/ambari-common/src/main/python/resource_management/core/resources/system.py
index 49e3a7b..a86fa54 100644
--- a/ambari-common/src/main/python/resource_management/core/resources/system.py
+++ b/ambari-common/src/main/python/resource_management/core/resources/system.py
@@ -74,6 +74,13 @@ class Execute(Resource):
command = ResourceArgument(default=lambda obj: obj.name)
creates = ResourceArgument()
+ """
+ cwd won't work for:
+ - commands run as sudo
+ - commands run as user (which uses sudo as well)
+
+ This is because non-interactive sudo commands doesn't support that.
+ """
cwd = ResourceArgument()
# this runs command with a specific env variables, env={'JAVA_HOME': '/usr/jdk'}
environment = ResourceArgument(default={})
@@ -102,9 +109,15 @@ class Execute(Resource):
- try_sleep
"""
wait_for_finish = BooleanArgument(default=True)
+ output_file = ResourceArgument()
+ """
+ For calling more advanced commands use as_sudo(command) option.
+ Example:
+ command1 = as_sudo(["cat,"/etc/passwd"]) + " | grep user"
+ command2 = as_sudo(["ls", "/root/example.txt") + " && " + as_sudo(["rm","-f","example.txt"])
+ """
sudo = BooleanArgument(default=False)
-
class ExecuteScript(Resource):
action = ForcedListArgument(default="run")
code = ResourceArgument(required=True)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py
index 2a472f2..7dbdc79 100644
--- a/ambari-common/src/main/python/resource_management/core/shell.py
+++ b/ambari-common/src/main/python/resource_management/core/shell.py
@@ -21,7 +21,7 @@ Ambari Agent
"""
import os
-__all__ = ["checked_call", "call", "quote_bash_args"]
+__all__ = ["checked_call", "call", "quote_bash_args", "as_user", "as_sudo"]
import string
import subprocess
@@ -31,16 +31,18 @@ from exceptions import Fail
from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
+SUDO_ENVIRONMENT_PLACEHOLDER = "{ENV_PLACEHOLDER}"
+
def checked_call(command, logoutput=False,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False):
- return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, sudo)
+ cwd=None, env={}, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, output_file=None, sudo=False):
+ return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, output_file, sudo)
def call(command, logoutput=False,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False):
- return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, sudo)
+ cwd=None, env={}, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, output_file=None, sudo=False):
+ return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, path, output_file, sudo)
def _call(command, logoutput=False, throw_on_failure=True,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False):
+ cwd=None, env={}, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, output_file=None, sudo=False):
"""
Execute shell command
@@ -53,22 +55,32 @@ def _call(command, logoutput=False, throw_on_failure=True,
"""
# convert to string and escape
if isinstance(command, (list, tuple)):
- command = ' '.join(quote_bash_args(x) for x in command)
+ command = string_cmd_from_args_list(command)
+ elif sudo:
+ # Since ambari user sudoer privileges may be restricted,
+ # without having /bin/bash permission.
+ # Running interpreted shell commands in scope of 'sudo' is not possible.
+ #
+ # In that case while passing string,
+ # any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
+ #
+ # In case of need to create more complicated commands with sudo use as_sudo(command) function.
+ err_msg = Logger.get_protected_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command))
+ raise Fail(err_msg)
# In case we will use sudo, we have to put all the environment inside the command,
# since Popen environment gets reset within sudo.
- export_command = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, 'export') + '; ' if env else ''
-
+ environment_str = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env,'')
+ command = command.replace(SUDO_ENVIRONMENT_PLACEHOLDER, environment_str, 1) # replace placeholder from as_sudo / as_user if present
+
+ bash_run_command = command if not sudo else "/usr/bin/sudo {0} -Hi {1}".format(environment_str, command)
+
if user:
- bash_run_command = "/usr/bin/sudo -Hsu {0} <<< {1}".format(quote_bash_args(user), quote_bash_args(export_command + command))
- # Go to home directory. In case we are in folder, which user cannot open, we might run into troubles with some utils
- cwd = os.path.expanduser('~'+user) if not cwd and os.path.exists(os.path.expanduser('~'+user)) else cwd
- elif sudo:
- bash_run_command = "/usr/bin/sudo -s <<< {0}".format(quote_bash_args(export_command + command))
+ # Outter environment gets reset within su. That's why we can't use environment passed to Popen.
+ su_export_command = "export {0} ; ".format(environment_str) if environment_str else ""
+ subprocess_command = ["/usr/bin/sudo","-Hi","su", "-", user, "-s", "/bin/bash", "-c", su_export_command + bash_run_command]
else:
- bash_run_command = command
-
- subprocess_command = ["/bin/bash","--login","-c", bash_run_command]
+ subprocess_command = ["/bin/bash","--login","-c", bash_run_command]
proc = subprocess.Popen(subprocess_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False,
@@ -102,6 +114,44 @@ def _call(command, logoutput=False, throw_on_failure=True,
return code, out
+def as_sudo(command, env=SUDO_ENVIRONMENT_PLACEHOLDER):
+ """
+ command - list or tuple of arguments.
+ env - when run as part of Execute resource, this SHOULD NOT be used.
+ It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if
+ """
+ if isinstance(command, (list, tuple)):
+ command = string_cmd_from_args_list(command)
+ else:
+ # Since ambari user sudoer privileges may be restricted,
+ # without having /bin/bash permission, and /bin/su permission.
+ # Running interpreted shell commands in scope of 'sudo' is not possible.
+ #
+ # In that case while passing string,
+ # any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
+ err_msg = Logger.get_protected_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple/list of arguments") % (command))
+ raise Fail(err_msg)
+
+ if env != SUDO_ENVIRONMENT_PLACEHOLDER:
+ env = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '')
+
+ return "/usr/bin/sudo {0} -Hi {1}".format(env, command)
+
+def as_user(command, user , env=SUDO_ENVIRONMENT_PLACEHOLDER):
+ if isinstance(command, (list, tuple)):
+ command = string_cmd_from_args_list(command)
+
+ if env != SUDO_ENVIRONMENT_PLACEHOLDER:
+ env = reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '')
+
+ export_command = "export {0} ; ".format(env)
+
+ result_command = "/usr/bin/sudo -Hi su - {0} -s /bin/bash -c {1}".format(user, quote_bash_args(export_command + command))
+ return result_command
+
+def string_cmd_from_args_list(command):
+ return ' '.join(quote_bash_args(x) for x in command)
+
def on_timeout(proc, q):
q.put(True)
if proc.poll() == None:
@@ -117,4 +167,4 @@ def quote_bash_args(command):
for char in command:
if char not in valid:
return "'" + command.replace("'", "'\"'\"'") + "'"
- return command
+ return command
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/core/sudo.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py
index 31cd468..fb81666 100644
--- a/ambari-common/src/main/python/resource_management/core/sudo.py
+++ b/ambari-common/src/main/python/resource_management/core/sudo.py
@@ -28,7 +28,7 @@ def chown(path, owner, group):
if owner:
shell.checked_call(["chown", owner, path], sudo=True)
if group:
- shell.checked_call(["chgrp", owner, path], sudo=True)
+ shell.checked_call(["chgrp", group, path], sudo=True)
# os.chmod replacement
def chmod(path, mode):
http://git-wip-us.apache.org/repos/asf/ambari/blob/5d07646e/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
index 19f4669..d854c55 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -25,6 +25,7 @@ from resource_management import *
class CopyFromLocalProvider(Provider):
def action_run(self):
+
path = self.resource.path
dest_dir = self.resource.dest_dir
dest_file = self.resource.dest_file
@@ -45,8 +46,8 @@ class CopyFromLocalProvider(Provider):
copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
dest_path = dest_dir + os.sep + dest_file_name
# Need to run unless as resource user
- su_cmd = 'su - {0} -c'.format(owner)
- unless_cmd = format("{su_cmd} '{kinnit_if_needed} export PATH=$PATH:{bin_dir} ; hadoop fs -ls {dest_path}' >/dev/null 2>&1")
+ unless_cmd = as_user("{kinnit_if_needed} ; hadoop fs -ls {dest_path}", owner, env={'PATH':bin_dir})
+
ExecuteHadoop(copy_cmd,
not_if=unless_cmd,