You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2014/10/13 20:07:39 UTC
git commit: AMBARI-7762. Confirm Hosts page is lagging.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-1.7.0 f2e62d3a6 -> 0e0e51fba
AMBARI-7762. Confirm Hosts page is lagging.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e0e51fb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e0e51fb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e0e51fb
Branch: refs/heads/branch-1.7.0
Commit: 0e0e51fba62710666a512dc33d81e50f5f8f1d1c
Parents: f2e62d3
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Mon Oct 13 19:52:31 2014 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Mon Oct 13 19:52:31 2014 +0300
----------------------------------------------------------------------
.../src/main/python/ambari_agent/HostCleanup.py | 51 +++++++++++++++++---
.../test/python/ambari_agent/TestHostCleanup.py | 35 +++++++++++---
2 files changed, 73 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e0e51fb/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 0840378..570c320 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -27,16 +27,16 @@ import subprocess
import logging
import shutil
import platform
+import fnmatch
import ConfigParser
import optparse
import shlex
import datetime
-import AmbariConfig
+from AmbariConfig import AmbariConfig
from pwd import getpwnam
from ambari_commons import OSCheck
logger = logging.getLogger()
-configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
PACKAGE_ERASE_CMD = {
"redhat": "yum erase -y {0}",
@@ -67,6 +67,9 @@ REPOS_KEY = "repo_list"
DIR_SECTION = "directories"
ADDITIONAL_DIRS = "additional_directories"
DIR_KEY = "dir_list"
+CACHE_FILES_PATTERN = {
+ 'alerts': ['*.json']
+}
PROCESS_SECTION = "processes"
PROCESS_KEY = "proc_list"
ALT_SECTION = "alternatives"
@@ -86,9 +89,9 @@ PACKAGES_BLACK_LIST = ["ambari-server", "ambari-agent"]
class HostCleanup:
def resolve_ambari_config(self):
try:
- config = AmbariConfig.AmbariConfig()
- if os.path.exists(configFile):
- config.read(configFile)
+ config = AmbariConfig()
+ if os.path.exists(AmbariConfig.CONFIG_FILE):
+ config.read(AmbariConfig.CONFIG_FILE)
else:
raise Exception("No config found, use default")
@@ -229,6 +232,33 @@ class HostCleanup:
return out
+
+ def do_clear_cache(self, cache_root, dir_map=None):
+ """
+ Clear cache dir according to provided root directory
+
+ cache_root - root dir for cache directory
+ dir_map - should be used only for recursive calls
+ """
+ global CACHE_FILES_PATTERN
+ file_map = CACHE_FILES_PATTERN if dir_map is None else dir_map
+ remList = []
+
+ # Build remove list according to masks
+ for folder in file_map:
+ if isinstance(file_map[folder], list): # here is list of file masks/files
+ for mask in file_map[folder]:
+ remList += self.get_files_in_dir("%s/%s" % (cache_root, folder), mask)
+ elif isinstance(file_map[folder], dict): # here described sub-folder
+ remList += self.do_clear_cache("%s/%s" % (cache_root, folder), file_map[folder])
+
+ if dir_map is not None: # push result list back as this is call from stack
+ return remList
+ else: # root call, so we have final list
+ self.do_erase_files_silent(remList)
+
+
+
# Alternatives exist as a stack of symlinks under /var/lib/alternatives/$name
# Script expects names of the alternatives as input
# We find all the symlinks using command, #] alternatives --display $name
@@ -276,7 +306,7 @@ class HostCleanup:
logger.error("Unable to kill process with pid: " + pid + ", " + stderrdata)
return 0
- def get_files_in_dir(self, dirPath):
+ def get_files_in_dir(self, dirPath, filemask = None):
fileList = []
if dirPath:
if os.path.exists(dirPath):
@@ -285,7 +315,11 @@ class HostCleanup:
for link in listdir:
path = dirPath + os.sep + link
if not os.path.islink(path) and not os.path.isdir(path):
- fileList.append(path)
+ if filemask is not None:
+ if fnmatch.fnmatch(path, filemask):
+ fileList.append(path)
+ else:
+ fileList.append(path)
return fileList
@@ -532,6 +566,9 @@ def main():
if propMap:
h.do_cleanup(propMap)
+ if os.path.exists(config.get('agent', 'cache_dir')):
+ h.do_clear_cache(config.get('agent', 'cache_dir'))
+
logger.info('Clean-up completed. The output is at %s' % (str(options.outputfile)))
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e0e51fb/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
index cdd5c36..c958412 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
@@ -117,6 +117,7 @@ class TestHostCleanup(TestCase):
self.silent = silent
self.java_home = java_home
+ @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
@patch.object(HostCleanup, 'get_YN_input')
@patch.object(HostCleanup.HostCleanup, 'do_cleanup')
@patch.object(HostCleanup.HostCleanup, 'is_current_user_root')
@@ -126,7 +127,7 @@ class TestHostCleanup(TestCase):
@patch.object(logging, 'FileHandler')
@patch.object(optparse.OptionParser, 'parse_args')
def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
- set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock):
+ set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock):
parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False,
False, 'java_home'), [])
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
@@ -144,7 +145,27 @@ class TestHostCleanup(TestCase):
read_host_check_file_mock.assert_called_with('/someinputfile')
self.assertTrue(get_yn_input_mock.called)
+ @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir')
+ @patch.object(HostCleanup.HostCleanup, 'do_erase_files_silent')
+ def test_clear_cache(self, erase_files_mock, get_files_mock):
+ old_data = HostCleanup.CACHE_FILES_PATTERN
+
+ HostCleanup.CACHE_FILES_PATTERN = {
+ 'somedir': ['*.txt']
+ }
+
+ files_list = ['/tmp/somedir/test.txt']
+ get_files_mock.return_value = files_list
+
+ self.hostcleanup.do_clear_cache('/tmp')
+
+ get_files_mock.assert_called_with('/tmp/somedir', '*.txt')
+ erase_files_mock.assert_called_with(files_list)
+
+ HostCleanup.CACHE_FILES_PATTERN = old_data
+
+ @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
@patch.object(HostCleanup, 'get_YN_input')
@patch.object(HostCleanup.HostCleanup, 'do_cleanup')
@patch.object(HostCleanup.HostCleanup, 'is_current_user_root')
@@ -154,7 +175,7 @@ class TestHostCleanup(TestCase):
@patch.object(logging, 'FileHandler')
@patch.object(optparse.OptionParser, 'parse_args')
def test_options_silent(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
- set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock):
+ set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock, clear_cache_mock):
parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile', '', False,
True, 'java_home'), [])
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
@@ -172,6 +193,7 @@ class TestHostCleanup(TestCase):
read_host_check_file_mock.assert_called_with('/someinputfile')
self.assertFalse(get_yn_input_mock.called)
+ @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
@patch.object(HostCleanup.HostCleanup, 'get_additional_dirs')
@patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives')
@patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')
@@ -185,7 +207,7 @@ class TestHostCleanup(TestCase):
do_erase_dir_silent_method,
do_erase_files_silent_method, do_kill_processes_method,
get_os_type_method, find_repo_files_for_repos_method,
- do_erase_alternatives_method, get_additional_dirs_method):
+ do_erase_alternatives_method, get_additional_dirs_method, clear_cache_mock):
out = StringIO.StringIO()
sys.stdout = out
get_additional_dirs_method.return_value = ['/tmp/hadoop-nagios','/tmp/hsperfdata_007']
@@ -216,7 +238,7 @@ class TestHostCleanup(TestCase):
sys.stdout = sys.__stdout__
-
+ @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
@patch.object(HostCleanup.HostCleanup, 'do_delete_by_owner')
@patch.object(HostCleanup.HostCleanup, 'get_user_ids')
@patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives')
@@ -232,7 +254,7 @@ class TestHostCleanup(TestCase):
do_erase_files_silent_method, do_kill_processes_method,
get_os_type_method, find_repo_files_for_repos_method,
do_erase_alternatives_method, get_user_ids_method,
- do_delete_by_owner_method):
+ do_delete_by_owner_method, clear_cache_mock):
global SKIP_LIST
oldSkipList = HostCleanup.SKIP_LIST
@@ -260,6 +282,7 @@ class TestHostCleanup(TestCase):
HostCleanup.SKIP_LIST = oldSkipList
sys.stdout = sys.__stdout__
+ @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
@patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')
@patch.object(OSCheck, "get_os_type")
@patch.object(HostCleanup.HostCleanup, 'do_kill_processes')
@@ -271,7 +294,7 @@ class TestHostCleanup(TestCase):
do_delete_users_method,
do_erase_dir_silent_method,
do_erase_files_silent_method, do_kill_processes_method,
- get_os_type_method, find_repo_files_for_repos_method):
+ get_os_type_method, find_repo_files_for_repos_method, clear_cache_mock):
out = StringIO.StringIO()
sys.stdout = out