You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2013/08/29 00:01:19 UTC
git commit: AMBARI-3047. Enhance host clean up to handle tmp files
and folders.
Updated Branches:
refs/heads/trunk 6028540df -> 4c2e398e0
AMBARI-3047. Enhance host clean up to handle tmp files and folders.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/4c2e398e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/4c2e398e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/4c2e398e
Branch: refs/heads/trunk
Commit: 4c2e398e0b2123048c7f8c3af9b49e82aec1562c
Parents: 6028540
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 28 15:01:02 2013 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Aug 28 15:01:02 2013 -0700
----------------------------------------------------------------------
.../ambari_agent/HostCheckReportFileHandler.py | 3 ++
.../src/main/python/ambari_agent/HostCleanup.py | 48 +++++++++++++++++++-
.../src/main/python/ambari_agent/HostInfo.py | 20 +-------
.../python/TestHostCheckReportFileHandler.py | 3 ++
ambari-agent/src/test/python/TestHostCleanup.py | 38 ++++++++++++++--
ambari-agent/src/test/python/TestHostInfo.py | 12 -----
6 files changed, 89 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
index be0de89..d46ab64 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
@@ -47,10 +47,13 @@ class HostCheckReportFileHandler:
if 'existingUsers' in hostInfo.keys():
items = []
+ items2 = []
for itemDetail in hostInfo['existingUsers']:
items.append(itemDetail['name'])
+ items2.append(itemDetail['homeDir'])
config.add_section('users')
config.set('users', 'usr_list', ','.join(items))
+ config.set('users', 'usr_homedir_list', ','.join(items2))
if 'alternatives' in hostInfo.keys():
items = []
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 409a115..7b63707 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -30,6 +30,7 @@ import shlex
import sys
import datetime
import AmbariConfig
+from pwd import getpwnam
logger = logging.getLogger()
configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
@@ -37,6 +38,7 @@ configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
PACKAGE_ERASE_CMD_RHEL = "yum erase -y {0}"
PACKAGE_ERASE_CMD_SUSE = "zypper -n -q remove {0}"
USER_ERASE_CMD = "userdel -rf {0}"
+GROUP_ERASE_CMD = "groupdel {0}"
PROC_KILL_CMD = "kill -9 {0}"
ALT_DISP_CMD = "alternatives --display {0}"
ALT_ERASE_CMD = "alternatives --remove {0} {1}"
@@ -51,6 +53,8 @@ PACKAGE_SECTION = "packages"
PACKAGE_KEY = "pkg_list"
USER_SECTION = "users"
USER_KEY = "usr_list"
+USER_HOMEDIR_KEY = "usr_homedir_list"
+USER_HOMEDIR_SECTION = "usr_homedir"
REPO_SECTION = "repositories"
REPOS_KEY = "repo_list"
DIR_SECTION = "directories"
@@ -59,6 +63,8 @@ PROCESS_SECTION = "processes"
PROCESS_KEY = "proc_list"
ALT_SECTION = "alternatives"
ALT_KEYS = ["symlink_list", "target_list"]
+HADOOP_GROUP = "hadoop"
+FOLDER_LIST = ["/tmp"]
# resources that should not be cleaned
REPOSITORY_BLACK_LIST = ["ambari.repo"]
@@ -83,11 +89,14 @@ class HostCleanup:
if argMap:
packageList = argMap.get(PACKAGE_SECTION)
userList = argMap.get(USER_SECTION)
+ homeDirList = argMap.get(USER_HOMEDIR_SECTION)
dirList = argMap.get(DIR_SECTION)
repoList = argMap.get(REPO_SECTION)
procList = argMap.get(PROCESS_SECTION)
alt_map = argMap.get(ALT_SECTION)
+ if userList and not USER_SECTION in SKIP_LIST:
+ userIds = self.get_user_ids(userList)
if procList and not PROCESS_SECTION in SKIP_LIST:
logger.info("\n" + "Killing pid's: " + str(procList) + "\n")
self.do_kill_processes(procList)
@@ -97,6 +106,8 @@ class HostCleanup:
if userList and not USER_SECTION in SKIP_LIST:
logger.info("\n" + "Deleting users: " + str(userList))
self.do_delete_users(userList)
+ self.do_erase_dir_silent(homeDirList)
+ self.do_delete_by_owner(userIds, FOLDER_LIST)
if dirList and not DIR_SECTION in SKIP_LIST:
logger.info("\n" + "Deleting directories: " + str(dirList))
self.do_erase_dir_silent(dirList)
@@ -139,6 +150,12 @@ class HostCleanup:
logger.warn("Cannot read user list: " + str(sys.exc_info()[0]))
try:
+ if config.has_option(USER_SECTION, USER_HOMEDIR_KEY):
+ propertyMap[USER_HOMEDIR_SECTION] = config.get(USER_SECTION, USER_HOMEDIR_KEY).split(',')
+ except:
+ logger.warn("Cannot read user homedir list: " + str(sys.exc_info()[0]))
+
+ try:
if config.has_option(REPO_SECTION, REPOS_KEY):
propertyMap[REPO_SECTION] = config.get(REPO_SECTION, REPOS_KEY).split(',')
except:
@@ -327,6 +344,34 @@ class HostCleanup:
logger.info("File doesn't exists: " + path)
return 0
+ def do_delete_group(self):
+ groupDelCommand = GROUP_ERASE_CMD.format(HADOOP_GROUP)
+ (returncode, stdoutdata, stderrdata) = self.run_os_command(groupDelCommand)
+ if returncode != 0:
+ logger.warn("Cannot delete group : " + HADOOP_GROUP + ", " + stderrdata)
+ else:
+ logger.info("Successfully deleted group: " + HADOOP_GROUP)
+
+ def do_delete_by_owner(self, userIds, folders):
+ for folder in folders:
+ for filename in os.listdir(folder):
+ fileToCheck = os.path.join(folder, filename)
+ stat = os.stat(fileToCheck)
+ if stat.st_uid in userIds:
+ self.do_erase_dir_silent([fileToCheck])
+ logger.info("Deleting file/folder: " + fileToCheck)
+
+ def get_user_ids(self, userList):
+ userIds = []
+ if userList:
+ for user in userList:
+ if user:
+ try:
+ userIds.append(getpwnam(user).pw_uid)
+ except Exception:
+ logger.warn("Cannot find user : " + user)
+ return userIds
+
def do_delete_users(self, userList):
if userList:
for user in userList:
@@ -337,6 +382,7 @@ class HostCleanup:
logger.warn("Cannot delete user : " + user + ", " + stderrdata)
else:
logger.info("Successfully deleted user: " + user)
+ self.do_delete_group()
return 0
def is_current_user_root(self):
@@ -401,7 +447,7 @@ def main():
help="log file to store results.", metavar="FILE")
parser.add_option("-k", "--skip", dest="skip",
help="(packages|users|directories|repositories|processes|alternatives)." +\
- " separator = ,")
+ " Use , as separator. Use empty string to clean all (by default users are skipped)")
(options, args) = parser.parse_args()
# set output file
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index 4805717..00b77ff 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -41,7 +41,8 @@ class HostInfo:
"hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios",
"oozie", "sqoop", "hue", "zookeeper", "mapred", "hdfs", "flume",
"ambari_qa", "hadoop_deploy", "rrdcached", "hcat", "ambari-qa",
- "sqoop-ambari-qa", "sqoop-ambari_qa"
+ "sqoop-ambari-qa", "sqoop-ambari_qa", "webhcat", "hadoop-hdfs", "hadoop-yarn",
+ "hadoop-mapreduce"
]
# List of live services checked for on the host, takes a map of plan strings
@@ -219,22 +220,6 @@ class HostInfo:
except:
pass
- def checkFoldersBasedOnNames(self, basePaths, projectNames, existingUsers, dirs):
- foldersToIgnore = []
- for user in existingUsers:
- foldersToIgnore.append(user['homeDir'])
- try:
- for dirName in basePaths:
- for project in projectNames:
- path = dirName.strip() + project.strip()
- if not path in foldersToIgnore and os.path.exists(path):
- obj = {}
- obj['type'] = self.dirType(path)
- obj['name'] = path
- dirs.append(obj)
- except:
- pass
-
def javaProcs(self, list):
try:
pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
@@ -321,7 +306,6 @@ class HostInfo:
dirs = []
self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
- self.checkFoldersBasedOnNames(self.DIRNAME_PATTERNS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
dict['stackFoldersAndFiles'] = dirs
installedPackages = []
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py b/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py
index 6ace6e4..1256717 100644
--- a/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py
+++ b/ambari-agent/src/test/python/TestHostCheckReportFileHandler.py
@@ -71,6 +71,7 @@ class TestHostCheckReportFileHandler(TestCase):
configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
configValidator.read(configPath)
users = configValidator.get('users', 'usr_list')
+ users = configValidator.get('users', 'usr_homedir_list')
self.assertEquals(users, '')
names = configValidator.get('alternatives', 'symlink_list')
targets = configValidator.get('alternatives', 'target_list')
@@ -124,7 +125,9 @@ class TestHostCheckReportFileHandler(TestCase):
configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
configValidator.read(configPath)
users = configValidator.get('users', 'usr_list')
+ homedirs = configValidator.get('users', 'usr_homedir_list')
self.assertEquals(users, 'user1')
+ self.assertEquals(homedirs, '/var/log')
names = configValidator.get('alternatives', 'symlink_list')
targets = configValidator.get('alternatives', 'target_list')
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/test/python/TestHostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestHostCleanup.py b/ambari-agent/src/test/python/TestHostCleanup.py
index 0bb513a..a3928be 100644
--- a/ambari-agent/src/test/python/TestHostCleanup.py
+++ b/ambari-agent/src/test/python/TestHostCleanup.py
@@ -42,6 +42,7 @@ PROCESS_KEY = "proc_list"
ALT_SECTION = "alternatives"
ALT_KEYS = ["symlink_list", "target_list"]
ALT_ERASE_CMD = "alternatives --remove {0} {1}"
+USER_HOMEDIR_SECTION = "usr_homedir"
class TestHostCleanup(TestCase):
@@ -148,7 +149,7 @@ created = 2013-07-02 20:39:22.162757"""
REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'],
PROCESS_SECTION:['abcd', 'pqrst'],
ALT_SECTION:{ALT_KEYS[0]:['alt1','alt2'], ALT_KEYS[1]:[
- 'dir1']}}
+ 'dir1']}, USER_HOMEDIR_SECTION:['decf']}
get_os_type_method.return_value = 'redhat'
find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst']
@@ -160,7 +161,8 @@ created = 2013-07-02 20:39:22.162757"""
self.assertTrue(do_erase_packages_method.called)
self.assertTrue(do_kill_processes_method.called)
self.assertTrue(do_erase_alternatives_method.called)
- do_erase_dir_silent_method.assert_called_once_with(['abcd', 'pqrst'])
+ calls = [call(['decf']), call(['abcd', 'pqrst'])]
+ do_erase_dir_silent_method.assert_has_calls(calls)
do_erase_packages_method.assert_called_once_with(['abcd', 'pqrst'])
do_erase_files_silent_method.assert_called_once_with(['abcd', 'pqrst'])
do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
@@ -172,6 +174,8 @@ created = 2013-07-02 20:39:22.162757"""
sys.stdout = sys.__stdout__
+ @patch.object(HostCleanup.HostCleanup, 'do_delete_by_owner')
+ @patch.object(HostCleanup.HostCleanup, 'get_user_ids')
@patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives')
@patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')
@patch.object(HostCleanup.HostCleanup, 'get_os_type')
@@ -184,7 +188,8 @@ created = 2013-07-02 20:39:22.162757"""
do_erase_dir_silent_method,
do_erase_files_silent_method, do_kill_processes_method,
get_os_type_method, find_repo_files_for_repos_method,
- do_erase_alternatives_method):
+ do_erase_alternatives_method, get_user_ids_method,
+ do_delete_by_owner_method):
out = StringIO.StringIO()
sys.stdout = out
@@ -198,6 +203,8 @@ created = 2013-07-02 20:39:22.162757"""
self.hostcleanup.do_cleanup(propertyMap)
+ self.assertFalse(do_delete_by_owner_method.called)
+ self.assertFalse(get_user_ids_method.called)
self.assertFalse(do_delete_users_method.called)
self.assertTrue(do_erase_dir_silent_method.called)
self.assertTrue(do_erase_files_silent_method.called)
@@ -235,12 +242,35 @@ created = 2013-07-02 20:39:22.162757"""
self.assertFalse(do_erase_files_silent_method.called)
self.assertFalse(do_erase_packages_method.called)
self.assertTrue(do_kill_processes_method.called)
- do_erase_dir_silent_method.assert_called_once_with(['abcd', 'pqrst'])
+ calls = [call(None), call(['abcd', 'pqrst'])]
+ do_erase_dir_silent_method.assert_has_calls(calls)
do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst'])
sys.stdout = sys.__stdout__
+ @patch.object(HostCleanup.HostCleanup, 'do_erase_dir_silent')
+ @patch("os.stat")
+ @patch("os.path.join")
+ @patch("os.listdir")
+ def test_do_delete_by_owner(self, listdir_mock, join_mock, stat_mock, do_erase_dir_silent_method):
+ listdir_mock.return_value = ["k", "j"]
+ join_mock.return_value = "path"
+ response = MagicMock()
+ response.st_uid = 1
+ stat_mock.return_value = response
+ self.hostcleanup.do_delete_by_owner([1, 2], ["a"])
+ self.assertTrue(do_erase_dir_silent_method.called)
+ calls = [call(["path"]), call(["path"])]
+ do_erase_dir_silent_method.assert_has_calls(calls)
+
+ @patch.object(HostCleanup.HostCleanup, 'run_os_command')
+ def test_do_delete_users(self, run_os_command_mock):
+ run_os_command_mock.return_value = (1, "", "")
+ self.hostcleanup.do_delete_users(["a", "b"])
+ self.assertTrue(run_os_command_mock.called)
+ calls = [call('userdel -rf a'), call('userdel -rf b'), call('groupdel hadoop')]
+ run_os_command_mock.assert_has_calls(calls)
@patch("ConfigParser.RawConfigParser")
@patch("__builtin__.open")
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/4c2e398e/ambari-agent/src/test/python/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestHostInfo.py b/ambari-agent/src/test/python/TestHostInfo.py
index 778b9f3..3bda116 100644
--- a/ambari-agent/src/test/python/TestHostInfo.py
+++ b/ambari-agent/src/test/python/TestHostInfo.py
@@ -196,18 +196,6 @@ class TestHostInfo(TestCase):
packageAnalyzer.allInstalledPackages(installedPackages)
self.assertEqual(installedPackages, [])
- @patch('os.path.exists')
- def test_checkFoldersBasedOnNames(self, path_mock):
- path_mock.return_value = True
- hostInfo = HostInfo()
- results = []
- existingUsers = [{'name':'a1', 'homeDir':'/home/a1'}, {'name':'b1', 'homeDir':'/home/b1'}]
- hostInfo.checkFoldersBasedOnNames(["/etc/conf", "/var/lib_", "/home/"], ["a1", "b1"], existingUsers, results)
- self.assertEqual(4, len(results))
- names = [i['name'] for i in results]
- for item in ['/etc/confa1', '/var/lib_a1', '/etc/confb1', '/var/lib_b1']:
- self.assertTrue(item in names)
-
@patch('os.path.exists')
def test_checkFolders(self, path_mock):