You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2013/07/25 02:16:17 UTC
git commit: AMBARI-2724. Improve host check to look for packages that
are installed by HDP-UTILS-1.1.0.15. (smohanty)
Updated Branches:
refs/heads/trunk bbef80059 -> 66ad274e7
AMBARI-2724. Improve host check to look for packages that are installed by HDP-UTILS-1.1.0.15. (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/66ad274e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/66ad274e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/66ad274e
Branch: refs/heads/trunk
Commit: 66ad274e7700ccd8f6c05e873da9ffe0ca4cde3e
Parents: bbef800
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jul 24 17:15:31 2013 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Jul 24 17:15:31 2013 -0700
----------------------------------------------------------------------
.../src/main/python/ambari_agent/Heartbeat.py | 5 ++
.../src/main/python/ambari_agent/HostCleanup.py | 2 +-
.../src/main/python/ambari_agent/HostInfo.py | 75 +++++++++++---------
.../python/ambari_agent/PackagesAnalyzer.py | 18 +++--
.../src/main/python/ambari_agent/shell.py | 4 --
ambari-agent/src/test/python/TestHostInfo.py | 46 +++++-------
ambari-agent/src/test/python/TestShell.py | 6 --
7 files changed, 79 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
index 79064b5..d4e0484 100644
--- a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
+++ b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
@@ -64,6 +64,11 @@ class Heartbeat:
# There may be IN_PROGRESS tasks
commandsInProgress = True
pass
+
+ # For first request/heartbeat assume no components are mapped
+ if int(id) == 0:
+ componentsMapped = False
+
logger.info("Sending heartbeat with response id: " + str(id) + " and "
"timestamp: " + str(timestamp) +
". Command(s) in progress: " + repr(commandsInProgress) +
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 975d6a0..0208c36 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -421,7 +421,7 @@ def main():
if options.skip:
global SKIP_LIST
- SKIP_LIST = options.skip.splt(',')
+ SKIP_LIST = options.skip.split(',')
is_root = h.is_current_user_root()
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index df1f802..a3fe9ff 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -39,7 +39,8 @@ class HostInfo:
DEFAULT_PROJECT_NAMES = [
"hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios",
"oozie", "sqoop", "hue", "zookeeper", "mapred", "hdfs", "flume",
- "ambari_qa", "hadoop_deploy", "rrdcached", "hcat"
+ "ambari_qa", "hadoop_deploy", "rrdcached", "hcat", "ambari-qa",
+ "sqoop-ambari-qa", "sqoop-ambari_qa"
]
# List of live services checked for on the host
@@ -59,9 +60,14 @@ class HostInfo:
"hadoop", "zookeeper"
]
+ # Additional path patterns to find existing directory
+ DIRNAME_PATTERNS = [
+ "/tmp/hadoop-", "/tmp/hsperfdata_"
+ ]
+
# Default set of directories that are checked for existence of files and folders
DEFAULT_DIRS = [
- "/etc", "/var/run", "/var/log", "/usr/lib", "/var/lib", "/var/tmp", "/tmp", "/var"
+ "/etc", "/var/run", "/var/log", "/usr/lib", "/var/lib", "/var/tmp", "/tmp", "/var", "/hadoop"
]
# Packages that are used to find repos (then repos are used to find other packages)
@@ -71,14 +77,21 @@ class HostInfo:
# Additional packages to look for (search packages that start with these)
ADDITIONAL_PACKAGES = [
- "rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb", "libconfuse", "ambari-log4j"
+ "rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb", "libconfuse", "ambari-log4j",
+ "mysql", "hadoop", "zookeeper"
]
- # ignores packages from repos whose names start with these strings
+ # ignore packages from repos whose names start with these strings
IGNORE_PACKAGES_FROM_REPOS = [
"ambari", "installed"
]
+ # ignore required packages
+ IGNORE_PACKAGES = [
+ "epel-release"
+ ]
+
+ # ignore repos from the list of repos to be cleaned
IGNORE_REPOS = [
"ambari", "HDP-UTILS"
]
@@ -105,30 +118,6 @@ class HostInfo:
return 'file'
return 'unknown'
- def rpmInfo(self, rpmList):
- config = AmbariConfig.config
-
- try:
- for rpmName in config.get('heartbeat', 'rpms').split(','):
- rpmName = rpmName.strip()
- rpm = {}
- rpm['name'] = rpmName
-
- try:
- osStat = subprocess.Popen(["rpm", "-q", rpmName], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = osStat.communicate()
- if (0 != osStat.returncode or 0 == len(out.strip())):
- rpm['installed'] = False
- else:
- rpm['installed'] = True
- rpm['version'] = out.strip()
- except:
- rpm['available'] = False
-
- rpmList.append(rpm)
- except:
- pass
-
def hadoopVarRunCount(self):
if not os.path.exists('/var/run/hadoop'):
return 0
@@ -216,6 +205,22 @@ class HostInfo:
except:
pass
+ def checkFoldersBasedOnNames(self, basePaths, projectNames, existingUsers, dirs):
+ foldersToIgnore = []
+ for user in existingUsers:
+ foldersToIgnore.append(user['homeDir'])
+ try:
+ for dirName in basePaths:
+ for project in projectNames:
+ path = dirName.strip() + project.strip()
+ if not path in foldersToIgnore and os.path.exists(path):
+ obj = {}
+ obj['type'] = self.dirType(path)
+ obj['name'] = path
+ dirs.append(obj)
+ except:
+ pass
+
def javaProcs(self, list):
try:
pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
@@ -274,9 +279,7 @@ class HostInfo:
dict['hostHealth']['diskStatus'] = [self.osdiskAvailableSpace("/")]
- rpms = []
- self.rpmInfo(rpms)
- dict['rpms'] = rpms
+ dict['rpms'] = []
liveSvcs = []
self.checkLiveServices(self.DEFAULT_LIVE_SERVICES, liveSvcs)
@@ -304,6 +307,7 @@ class HostInfo:
dirs = []
self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
+ self.checkFoldersBasedOnNames(self.DIRNAME_PATTERNS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
dict['stackFoldersAndFiles'] = dirs
installedPackages = []
@@ -314,14 +318,15 @@ class HostInfo:
repos = []
self.packages.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
self.IGNORE_PACKAGES_FROM_REPOS, repos)
- repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
- dict['existingRepos'] = repos
-
- packagesInstalled = self.packages.getInstalledPkgsByRepo(repos, installedPackages)
+ packagesInstalled = self.packages.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages)
additionalPkgsInstalled = self.packages.getInstalledPkgsByNames(
self.ADDITIONAL_PACKAGES, installedPackages)
allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
dict['installedPackages'] = self.packages.getPackageDetails(installedPackages, allPackages)
+
+ repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
+ dict['existingRepos'] = repos
+
self.reportFileHandler.writeHostCheckFile(dict)
pass
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/main/python/ambari_agent/PackagesAnalyzer.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/PackagesAnalyzer.py b/ambari-agent/src/main/python/ambari_agent/PackagesAnalyzer.py
index b94c387..7cc8d75 100644
--- a/ambari-agent/src/main/python/ambari_agent/PackagesAnalyzer.py
+++ b/ambari-agent/src/main/python/ambari_agent/PackagesAnalyzer.py
@@ -31,7 +31,7 @@ logger = logging.getLogger()
class PackagesAnalyzer:
# default timeout for async invoked processes
- TIMEOUT_SECONDS = 60
+ TIMEOUT_SECONDS = 30
event = threading.Event()
def launch_subprocess(self, command):
@@ -41,7 +41,7 @@ class PackagesAnalyzer:
self.event.wait(self.TIMEOUT_SECONDS)
if command.returncode is None:
logger.error("Task timed out and will be killed")
- shell.killprocessgrp(command.pid)
+ shell.kill_process_with_children(command.pid)
pass
def subprocessWithTimeout(self, command):
@@ -175,15 +175,25 @@ class PackagesAnalyzer:
repoList.append(repo)
# Get all the installed packages from the repos listed in repos
- def getInstalledPkgsByRepo(self, repos, installedPackages):
+ def getInstalledPkgsByRepo(self, repos, ignorePackages, installedPackages):
packagesFromRepo = []
+ packagesToRemove = []
for repo in repos:
subResult = []
for item in installedPackages:
if repo == item[2]:
subResult.append(item[0])
packagesFromRepo = list(set(packagesFromRepo + subResult))
- return packagesFromRepo
+
+ for package in packagesFromRepo:
+ keepPackage = True
+ for ignorePackage in ignorePackages:
+ if self.nameMatch(ignorePackage, package):
+ keepPackage = False
+ break
+ if keepPackage:
+ packagesToRemove.append(package)
+ return packagesToRemove
# Gets all installed packages that start with names in pkgNames
def getInstalledPkgsByNames(self, pkgNames, installedPackages):
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/main/python/ambari_agent/shell.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/shell.py b/ambari-agent/src/main/python/ambari_agent/shell.py
index cf2b98e..95a16ad 100644
--- a/ambari-agent/src/main/python/ambari_agent/shell.py
+++ b/ambari-agent/src/main/python/ambari_agent/shell.py
@@ -59,10 +59,6 @@ def killstaleprocesses():
os.unlink(os.path.join(prefix,file))
logger.info ("Killed stale processes")
-
-def killprocessgrp(pid):
- run_kill_function(os.killpg, pid)
-
def kill_process_with_children(parent_pid):
def kill_tree_function(pid, signal):
'''
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/test/python/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestHostInfo.py b/ambari-agent/src/test/python/TestHostInfo.py
index 8839ff3..b45cd18 100644
--- a/ambari-agent/src/test/python/TestHostInfo.py
+++ b/ambari-agent/src/test/python/TestHostInfo.py
@@ -107,9 +107,9 @@ class TestHostInfo(TestCase):
for repo in expected:
self.assertTrue(repo in repos)
- packagesInstalled = packageAnalyzer.getInstalledPkgsByRepo(repos, installedPackages)
- self.assertEqual(7, len(packagesInstalled))
- expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "epel", "epel-2", "def-def.x86"]
+ packagesInstalled = packageAnalyzer.getInstalledPkgsByRepo(repos, ["epel"], installedPackages)
+ self.assertEqual(5, len(packagesInstalled))
+ expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "def-def.x86"]
for repo in expected:
self.assertTrue(repo in packagesInstalled)
@@ -121,8 +121,8 @@ class TestHostInfo(TestCase):
self.assertTrue(additionalPkg in additionalPkgsInstalled)
allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
- self.assertEqual(9, len(allPackages))
- expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "epel", "epel-2", "nagios", "rrd", "def-def.x86"]
+ self.assertEqual(7, len(allPackages))
+ expected = ["hadoop-a", "zk", "webhcat", "hadoop-b", "nagios", "rrd", "def-def.x86"]
for package in expected:
self.assertTrue(package in allPackages)
@@ -197,6 +197,19 @@ class TestHostInfo(TestCase):
self.assertEqual(installedPackages, [])
@patch('os.path.exists')
+ def test_checkFoldersBasedOnNames(self, path_mock):
+ path_mock.return_value = True
+ hostInfo = HostInfo()
+ results = []
+ existingUsers = [{'name':'a1', 'homeDir':'/home/a1'}, {'name':'b1', 'homeDir':'/home/b1'}]
+ hostInfo.checkFoldersBasedOnNames(["/etc/conf", "/var/lib_", "/home/"], ["a1", "b1"], existingUsers, results)
+ self.assertEqual(4, len(results))
+ names = [i['name'] for i in results]
+ for item in ['/etc/confa1', '/var/lib_a1', '/etc/confb1', '/var/lib_b1']:
+ self.assertTrue(item in names)
+
+
+ @patch('os.path.exists')
def test_checkFolders(self, path_mock):
path_mock.return_value = True
hostInfo = HostInfo()
@@ -459,26 +472,5 @@ class TestHostInfo(TestCase):
self.assertEquals(result[0]['target'], 'real_path_to_conf')
- @patch("subprocess.Popen")
- def test_rpmInfo(self, subproc_popen):
- hostInfo = HostInfo()
- p = MagicMock()
- p.returncode = 0
- p.communicate.side_effect = [('out', 'err'), ('', 'err') , ('out', 'err', 'fail')]
- subproc_popen.return_value = p
- rpmList = []
- hostInfo.rpmInfo(rpmList)
-
- self.assertEquals(rpmList[0]['version'], 'out')
- self.assertEquals(rpmList[0]['name'], 'glusterfs')
- self.assertTrue(rpmList[0]['installed'])
-
- self.assertEquals(rpmList[1]['name'], 'openssl')
- self.assertFalse(rpmList[1]['installed'])
-
- self.assertFalse(rpmList[2]['available'])
- self.assertEquals(rpmList[2]['name'], 'wget')
-
-
if __name__ == "__main__":
- unittest.main(verbosity=2)
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/66ad274e/ambari-agent/src/test/python/TestShell.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestShell.py b/ambari-agent/src/test/python/TestShell.py
index 50dd689..8c5311a 100644
--- a/ambari-agent/src/test/python/TestShell.py
+++ b/ambari-agent/src/test/python/TestShell.py
@@ -86,9 +86,3 @@ class TestShell(unittest.TestCase):
else:
# Do not run under other systems
pass
-
-
- @patch("os.killpg")
- def test_killprocessgrp(self, killpg_grp):
- shell.killprocessgrp(999)
- self.assertTrue(killpg_grp.called)
\ No newline at end of file