You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by st...@apache.org on 2016/03/03 16:37:55 UTC
ambari git commit: AMBARI-14690. Configurable system resource values
for ambari-agent. (Oliver Szabo via stoader)
Repository: ambari
Updated Branches:
refs/heads/trunk 236beac47 -> 492a30540
AMBARI-14690. Configurable system resource values for ambari-agent. (Oliver Szabo via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/492a3054
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/492a3054
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/492a3054
Branch: refs/heads/trunk
Commit: 492a305409a26f687e55be7be624b9e95ce422d7
Parents: 236beac
Author: Toader, Sebastian <st...@hortonworks.com>
Authored: Thu Mar 3 16:36:27 2016 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Thu Mar 3 16:36:27 2016 +0100
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 1 +
ambari-agent/conf/windows/ambari-agent.ini | 1 +
.../main/python/ambari_agent/AmbariConfig.py | 1 +
.../src/main/python/ambari_agent/Facter.py | 60 ++++++++++++++++++--
.../test/python/ambari_agent/TestHardware.py | 51 +++++++++++++++++
5 files changed, 110 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/492a3054/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index 80afdb2..b46bfe5 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -30,6 +30,7 @@ tolerate_download_failures=true
run_as_user=root
parallel_execution=0
alert_grace_period=5
+system_resource_overrides=/etc/resource_overrides
[security]
keysdir=/var/lib/ambari-agent/keys
http://git-wip-us.apache.org/repos/asf/ambari/blob/492a3054/ambari-agent/conf/windows/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/windows/ambari-agent.ini b/ambari-agent/conf/windows/ambari-agent.ini
index 1b24f25..e490f7c 100644
--- a/ambari-agent/conf/windows/ambari-agent.ini
+++ b/ambari-agent/conf/windows/ambari-agent.ini
@@ -30,6 +30,7 @@ cache_dir=cache
tolerate_download_failures=true
parallel_execution=0
alert_grace_period=5
+system_resource_overrides=\\etc\\resource_overrides
[security]
keysdir=keys
http://git-wip-us.apache.org/repos/asf/ambari/blob/492a3054/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index 2c82ca5..b1d4419 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -46,6 +46,7 @@ data_cleanup_max_size_MB = 100
ping_port=8670
cache_dir={ps}var{ps}lib{ps}ambari-agent{ps}cache
parallel_execution=0
+system_resource_overrides={ps}etc{ps}resource_overrides
[services]
http://git-wip-us.apache.org/repos/asf/ambari/blob/492a3054/ambari-agent/src/main/python/ambari_agent/Facter.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Facter.py b/ambari-agent/src/main/python/ambari_agent/Facter.py
index 85650b9..bad9b96 100644
--- a/ambari-agent/src/main/python/ambari_agent/Facter.py
+++ b/ambari-agent/src/main/python/ambari_agent/Facter.py
@@ -31,6 +31,8 @@ import subprocess
from ambari_commons.shell import shellRunner
import time
import uuid
+import json
+import glob
from AmbariConfig import AmbariConfig
from ambari_commons import OSCheck, OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
@@ -150,6 +152,47 @@ class Facter(object):
def getUptimeDays(self):
return self.getUptimeSeconds() / (60 * 60 * 24)
+ def getSystemResourceIfExists(self, systemResources, key, default):
+ if key in systemResources:
+ return systemResources[key]
+ else:
+ return default
+
+ def replaceFacterInfoWithSystemResources(self, systemResources, facterInfo):
+ """
+ Replace facter info with fake system resource data (if there are any).
+ """
+ for key in facterInfo:
+ facterInfo[key] = self.getSystemResourceIfExists(systemResources, key, facterInfo[key])
+ return facterInfo
+
+ def getSystemResourceOverrides(self):
+ """
+ Read all json files from 'system_resource_overrides' directory, and later these values are used as
+ fake system data for hosts. In case of the key-value pairs cannot be loaded use default behaviour.
+ """
+ systemResources = {}
+ if self.config.has_option('agent', 'system_resource_overrides'):
+ systemResourceDir = self.config.get('agent', 'system_resource_overrides', '').strip()
+ if systemResourceDir:
+ if os.path.isdir(systemResourceDir) and os.path.exists(systemResourceDir):
+ try:
+ for filename in glob.glob('%s/*.json' % systemResourceDir):
+ with open(filename) as fp:
+ data = json.loads(fp.read())
+ for (key, value) in data.items():
+ systemResources[key] = data[key]
+ except:
+ log.warn(
+ "Cannot read values from json files in %s. it won't be used for gathering system resources." % systemResourceDir)
+ else:
+ log.info(
+ "Directory: '%s' does not exist - it won't be used for gathering system resources." % systemResourceDir)
+ else:
+ log.info("'system_resource_dir' is not set - it won't be used for gathering system resources.")
+ return systemResources
+
+
def facterInfo(self):
facterInfo = {}
facterInfo['id'] = self.getId()
@@ -290,8 +333,12 @@ class FacterWindows(Facter):
def facterInfo(self):
facterInfo = super(FacterWindows, self).facterInfo()
- facterInfo['swapsize'] = Facter.convertSizeMbToGb(self.getSwapSize())
- facterInfo['swapfree'] = Facter.convertSizeMbToGb(self.getSwapFree())
+ systemResourceOverrides = self.getSystemResourceOverrides()
+ facterInfo = self.replaceFacterInfoWithSystemResources(systemResourceOverrides, facterInfo)
+ facterInfo['swapsize'] = Facter.convertSizeMbToGb(
+ self.getSystemResourceIfExists(systemResourceOverrides, 'swapsize', self.getSwapSize()))
+ facterInfo['swapfree'] = Facter.convertSizeMbToGb(
+ self.getSystemResourceIfExists(systemResourceOverrides, 'swapfree', self.getSwapFree()))
return facterInfo
@@ -468,9 +515,14 @@ class FacterLinux(Facter):
def facterInfo(self):
facterInfo = super(FacterLinux, self).facterInfo()
+ systemResourceOverrides = self.getSystemResourceOverrides()
+ facterInfo = self.replaceFacterInfoWithSystemResources(systemResourceOverrides, facterInfo)
+
facterInfo['selinux'] = self.isSeLinux()
- facterInfo['swapsize'] = Facter.convertSizeKbToGb(self.getSwapSize())
- facterInfo['swapfree'] = Facter.convertSizeKbToGb(self.getSwapFree())
+ facterInfo['swapsize'] = Facter.convertSizeKbToGb(
+ self.getSystemResourceIfExists(systemResourceOverrides, 'swapsize', self.getSwapSize()))
+ facterInfo['swapfree'] = Facter.convertSizeKbToGb(
+ self.getSystemResourceIfExists(systemResourceOverrides, 'swapfree', self.getSwapFree()))
return facterInfo
http://git-wip-us.apache.org/repos/asf/ambari/blob/492a3054/ambari-agent/src/test/python/ambari_agent/TestHardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
index 9ab02dc..fb390b2 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
@@ -29,6 +29,7 @@ from ambari_agent.Hardware import Hardware
from ambari_agent.AmbariConfig import AmbariConfig
from ambari_agent.Facter import Facter, FacterLinux
from ambari_commons import OSCheck
+from glob import glob
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(platform,"linux_distribution", new = ('Suse','11','Final'))
@@ -265,6 +266,56 @@ SwapFree: 1598676 kB
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'My_new_family')
+
+ @patch("os.path.exists")
+ @patch("os.path.isdir")
+ @patch("json.loads")
+ @patch("glob.glob")
+ @patch("__builtin__.open")
+ @patch.object(OSCheck, "get_os_type")
+ @patch.object(OSCheck, "get_os_version")
+ @patch.object(FacterLinux, "resolve_ambari_config")
+ def test_system_resource_overrides(self, resolve_ambari_config, get_os_version_mock, get_os_type_mock,
+ open_mock, glob_mock, json_mock, isdir, exists):
+ get_os_type_mock.return_value = "suse"
+ get_os_version_mock.return_value = "11"
+ config = MagicMock()
+ config.get.return_value = '/etc/custom_resource_overrides'
+ config.has_option.return_value = True
+ resolve_ambari_config.return_value = config
+ isdir.return_value = True
+ exists.return_value = True
+ open_mock.return_value.read = "1"
+ file_handle = open_mock.return_value.__enter__.return_value
+ file_handle.read.return_value = '1'
+ glob_mock.side_effect = \
+ [
+ [
+ "/etc/custom_resource_overrides/1.json",
+ "/etc/custom_resource_overrides/2.json"
+ ]
+ ]
+ json_data = json_mock.return_value
+ json_data.items.return_value = [('key', 'value')]
+ json_data.__getitem__.return_value = 'value'
+
+ facter = Facter()
+ facter.config = config
+ result = facter.getSystemResourceOverrides()
+
+ isdir.assert_called_with('/etc/custom_resource_overrides')
+ exists.assert_called_with('/etc/custom_resource_overrides')
+ glob_mock.assert_called_with('/etc/custom_resource_overrides/*.json')
+ self.assertTrue(config.has_option.called)
+ self.assertTrue(config.get.called)
+ self.assertTrue(glob_mock.called)
+ self.assertEquals(2, file_handle.read.call_count)
+ self.assertEquals(2, open_mock.call_count)
+ self.assertEquals(2, json_mock.call_count)
+ self.assertEquals('value', result['key'])
+
+
+
if __name__ == "__main__":
unittest.main()