You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/11/02 20:59:54 UTC
ambari git commit: AMBARI-18733. Perf: Allow running multiple Ambari
Agents on the same host (alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk fca6f1c3a -> 4e96cf51c
AMBARI-18733. Perf: Allow running multiple Ambari Agents on the same host (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4e96cf51
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4e96cf51
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4e96cf51
Branch: refs/heads/trunk
Commit: 4e96cf51c5a796c7efc8415c7106ca5d071678a2
Parents: fca6f1c
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Wed Nov 2 14:59:39 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Wed Nov 2 14:59:39 2016 -0700
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent | 12 +++++-
ambari-agent/conf/windows/service_wrapper.py | 1 +
.../main/python/ambari_agent/AmbariConfig.py | 44 +++++++++++++++-----
.../src/main/python/ambari_agent/Facter.py | 21 +++++++---
.../src/main/python/ambari_agent/Hardware.py | 8 ++--
.../src/main/python/ambari_agent/Heartbeat.py | 5 ++-
.../src/main/python/ambari_agent/Register.py | 2 +-
.../src/main/python/ambari_agent/main.py | 28 +++++++++++--
.../test/python/ambari_agent/TestCheckWebUI.py | 2 +
.../test/python/ambari_agent/TestHardware.py | 30 +++++++------
.../src/test/python/ambari_agent/TestMain.py | 4 +-
11 files changed, 116 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/conf/unix/ambari-agent
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent b/ambari-agent/conf/unix/ambari-agent
index 30897dd..0f791ca 100755
--- a/ambari-agent/conf/unix/ambari-agent
+++ b/ambari-agent/conf/unix/ambari-agent
@@ -33,9 +33,17 @@ case "${1:-}" in
;;
esac
+export HOME_DIR=""
+if [ "$#" = 3 ] && [ $2 = "--home" ] ; then
+ export HOME_DIR=$3
+ echo "Allow running multiple agents on this host; will use custom Home Dir: $HOME_DIR"
+fi
+
+export CONFIG_FILE="$HOME_DIR/etc/ambari-agent/conf/ambari-agent.ini"
+
get_agent_property() {
property_name="$1"
- value=$(awk -F "=" "/^$property_name/ {print \$2}" /etc/ambari-agent/conf/ambari-agent.ini)
+ value=$(awk -F "=" "/^$property_name/ {print \$2}" $CONFIG_FILE)
echo $value
}
@@ -51,7 +59,7 @@ valid_path() {
export PATH=/usr/sbin:/sbin:/usr/lib/ambari-server/*:$PATH
-export AMBARI_CONF_DIR=/etc/ambari-server/conf:$PATH
+export AMBARI_CONF_DIR=$HOME_DIR/etc/ambari-server/conf:$PATH
# Because Ambari rpm unpacks modules here on all systems
export PYTHONPATH=/usr/lib/python2.6/site-packages:${PYTHONPATH:-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/conf/windows/service_wrapper.py
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/windows/service_wrapper.py b/ambari-agent/conf/windows/service_wrapper.py
index d031c34..ac9e22d 100644
--- a/ambari-agent/conf/windows/service_wrapper.py
+++ b/ambari-agent/conf/windows/service_wrapper.py
@@ -124,6 +124,7 @@ def ctrlHandler(ctrlType):
#
def setup(options):
config = AmbariConfig()
+ # TODO AMBARI-18733, need to read home_dir to get correct config file.
configFile = config.getConfigFile()
updateConfigServerHostname(configFile, options.host_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index 78b34e5..6e4d74a 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -191,7 +191,7 @@ class AmbariConfig:
@staticmethod
@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
- def getConfigFile():
+ def getConfigFile(home_dir=""):
if 'AMBARI_AGENT_CONF_DIR' in os.environ:
return os.path.join(os.environ['AMBARI_AGENT_CONF_DIR'], "ambari-agent.ini")
else:
@@ -199,32 +199,56 @@ class AmbariConfig:
@staticmethod
@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
- def getConfigFile():
+ def getConfigFile(home_dir=""):
+ """
+ Get the configuration file path.
+ :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path.
+ :return: Configuration file path.
+ """
if 'AMBARI_AGENT_CONF_DIR' in os.environ:
return os.path.join(os.environ['AMBARI_AGENT_CONF_DIR'], "ambari-agent.ini")
else:
- return os.path.join(os.sep, "etc", "ambari-agent", "conf", "ambari-agent.ini")
+ # home_dir may be an empty string
+ return os.path.join(os.sep, home_dir, "etc", "ambari-agent", "conf", "ambari-agent.ini")
+ # TODO AMBARI-18733, change usages of this function to provide the home_dir.
@staticmethod
- def getLogFile():
+ def getLogFile(home_dir=""):
+ """
+ Get the log file path.
+ :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path.
+ :return: Log file path.
+ """
if 'AMBARI_AGENT_LOG_DIR' in os.environ:
return os.path.join(os.environ['AMBARI_AGENT_LOG_DIR'], "ambari-agent.log")
else:
- return os.path.join(os.sep, "var", "log", "ambari-agent", "ambari-agent.log")
-
+ return os.path.join(os.sep, home_dir, "var", "log", "ambari-agent", "ambari-agent.log")
+
+ # TODO AMBARI-18733, change usages of this function to provide the home_dir.
@staticmethod
- def getAlertsLogFile():
+ def getAlertsLogFile(home_dir=""):
+ """
+ Get the alerts log file path.
+ :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path.
+ :return: Alerts log file path.
+ """
if 'AMBARI_AGENT_LOG_DIR' in os.environ:
return os.path.join(os.environ['AMBARI_AGENT_LOG_DIR'], "ambari-agent.log")
else:
- return os.path.join(os.sep, "var", "log", "ambari-agent", "ambari-alerts.log")
+ return os.path.join(os.sep, home_dir, "var", "log", "ambari-agent", "ambari-alerts.log")
+ # TODO AMBARI-18733, change usages of this function to provide the home_dir.
@staticmethod
- def getOutFile():
+ def getOutFile(home_dir=""):
+ """
+ Get the out file path.
+ :param home_dir: In production, will be "". When running multiple Agents per host, each agent will have a unique path.
+ :return: Out file path.
+ """
if 'AMBARI_AGENT_LOG_DIR' in os.environ:
return os.path.join(os.environ['AMBARI_AGENT_LOG_DIR'], "ambari-agent.out")
else:
- return os.path.join(os.sep, "var", "log", "ambari-agent", "ambari-agent.out")
+ return os.path.join(os.sep, home_dir, "var", "log", "ambari-agent", "ambari-agent.out")
def has_option(self, section, option):
return self.config.has_option(section, option)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Facter.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Facter.py b/ambari-agent/src/main/python/ambari_agent/Facter.py
index a1f815d..3643ff7 100644
--- a/ambari-agent/src/main/python/ambari_agent/Facter.py
+++ b/ambari-agent/src/main/python/ambari_agent/Facter.py
@@ -53,10 +53,18 @@ def run_os_command(cmd):
class Facter(object):
- def __init__(self):
- self.config = self.resolve_ambari_config()
-
+ def __init__(self, config):
+ """
+ Initialize the configs, which can be provided if using multiple Agents per host.
+ :param config: Agent configs. None if will use the default location.
+ """
+ self.config = config if config is not None else self.resolve_ambari_config()
+
def resolve_ambari_config(self):
+ """
+ Resolve the default Ambari Agent configs.
+ :return: The default configs.
+ """
try:
config = AmbariConfig()
if os.path.exists(AmbariConfig.getConfigFile()):
@@ -370,8 +378,8 @@ class FacterLinux(Facter):
GET_UPTIME_CMD = "cat /proc/uptime"
GET_MEMINFO_CMD = "cat /proc/meminfo"
- def __init__(self):
- super(FacterLinux,self).__init__()
+ def __init__(self, config):
+ super(FacterLinux,self).__init__(config)
self.DATA_IFCONFIG_SHORT_OUTPUT = FacterLinux.setDataIfConfigShortOutput()
self.DATA_UPTIME_OUTPUT = FacterLinux.setDataUpTimeOutput()
self.DATA_MEMINFO_OUTPUT = FacterLinux.setMemInfoOutput()
@@ -547,7 +555,8 @@ class FacterLinux(Facter):
def main(argv=None):
- print Facter().facterInfo()
+ config = None
+ print Facter(config).facterInfo()
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Hardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Hardware.py b/ambari-agent/src/main/python/ambari_agent/Hardware.py
index a65c1cd..3c94d28 100644
--- a/ambari-agent/src/main/python/ambari_agent/Hardware.py
+++ b/ambari-agent/src/main/python/ambari_agent/Hardware.py
@@ -42,11 +42,12 @@ class Hardware:
IGNORE_ROOT_MOUNTS = ["proc", "dev", "sys"]
IGNORE_DEVICES = ["proc", "tmpfs", "cgroup", "mqueue", "shm"]
- def __init__(self):
+ def __init__(self, config):
self.hardware = {
'mounts': Hardware.osdisks()
}
- self.hardware.update(Facter().facterInfo())
+ self.config = config
+ self.hardware.update(Facter(self.config).facterInfo())
@classmethod
def _parse_df_line(cls, line):
@@ -169,7 +170,8 @@ def main():
from resource_management.core.logger import Logger
Logger.initialize_logger()
- print Hardware().get()
+ config = None
+ print Hardware(config).get()
if __name__ == '__main__':
main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
index 82ea9b6..1e05aae 100644
--- a/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
+++ b/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
@@ -111,8 +111,9 @@ def main(argv=None):
from ambari_agent.Controller import Controller
cfg = AmbariConfig()
- if os.path.exists(AmbariConfig.getConfigFile()):
- cfg.read(AmbariConfig.getConfigFile())
+ config_file_path = AmbariConfig.getConfigFile(home_dir="")
+ if os.path.exists(config_file_path):
+ cfg.read(config_file_path)
else:
raise Exception("No config found, use default")
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/Register.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Register.py b/ambari-agent/src/main/python/ambari_agent/Register.py
index 2d85b03..0c811c6 100644
--- a/ambari-agent/src/main/python/ambari_agent/Register.py
+++ b/ambari-agent/src/main/python/ambari_agent/Register.py
@@ -31,8 +31,8 @@ class Register:
""" Registering with the server. Get the hardware profile and
declare success for now """
def __init__(self, config):
- self.hardware = Hardware()
self.config = config
+ self.hardware = Hardware(self.config)
def build(self, version, id='-1'):
global clusterId, clusterDefinitionRevision, firstContact
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/main/python/ambari_agent/main.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/main.py b/ambari-agent/src/main/python/ambari_agent/main.py
index 8cbb839..04e0c2c 100644
--- a/ambari-agent/src/main/python/ambari_agent/main.py
+++ b/ambari-agent/src/main/python/ambari_agent/main.py
@@ -72,7 +72,12 @@ alerts_logger = logging.getLogger('ambari_alerts')
formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()
+
+# Global variables to be set later.
+home_dir = ""
+
config = AmbariConfig.AmbariConfig()
+# TODO AMBARI-18733, remove this global variable and calculate it based on home_dir once it is set.
configFile = config.getConfigFile()
two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY
@@ -110,7 +115,8 @@ def add_syslog_handler(logger):
def update_log_level(config):
# Setting loglevel based on config file
global logger
- log_cfg_file = os.path.join(os.path.dirname(AmbariConfig.AmbariConfig.getConfigFile()), "logging.conf")
+ global home_dir
+ log_cfg_file = os.path.join(os.path.dirname(AmbariConfig.AmbariConfig.getConfigFile(home_dir)), "logging.conf")
if os.path.exists(log_cfg_file):
logging.config.fileConfig(log_cfg_file)
# create logger
@@ -132,11 +138,15 @@ def update_log_level(config):
logger.info("Default loglevel=DEBUG")
-# ToDo: move that function inside AmbariConfig
+# TODO AMBARI-18733, move inside AmbariConfig
def resolve_ambari_config():
+ """
+ Load the configurations.
+ In production, home_dir will be "". When running multiple Agents per host, each agent will have a unique path.
+ """
global config
- configPath = os.path.abspath(AmbariConfig.AmbariConfig.getConfigFile())
-
+ global home_dir
+ configPath = os.path.abspath(AmbariConfig.AmbariConfig.getConfigFile(home_dir))
try:
if os.path.exists(configPath):
config.read(configPath)
@@ -254,9 +264,11 @@ def stop_agent():
sys.exit(0)
def reset_agent(options):
+ global home_dir
try:
# update agent config file
agent_config = ConfigParser.ConfigParser()
+ # TODO AMBARI-18733, calculate configFile based on home_dir
agent_config.read(configFile)
server_host = agent_config.get('server', 'hostname')
new_host = options[2]
@@ -302,13 +314,17 @@ def run_threads(server_hostname, heartbeat_stop_callback):
# we need this for windows os, where no sigterm available
def main(heartbeat_stop_callback=None):
global config
+ global home_dir
+
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="verbose log output", default=False)
parser.add_option("-e", "--expected-hostname", dest="expected_hostname", action="store",
help="expected hostname of current host. If hostname differs, agent will fail", default=None)
+ parser.add_option("--home", dest="home_dir", action="store", help="Home directory", default="")
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
+ home_dir = options.home_dir
logging_level = logging.DEBUG if options.verbose else logging.INFO
@@ -318,6 +334,10 @@ def main(heartbeat_stop_callback=None):
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
Logger.initialize_logger('resource_management', logging_level=logging_level)
+ if home_dir != "":
+ # When running multiple Ambari Agents on this host for simulation, each one will use a unique home directory.
+ Logger.info("Agent is using Home Dir: %s" % str(home_dir))
+
# use the host's locale for numeric formatting
try:
locale.setlocale(locale.LC_ALL, '')
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py b/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py
index 4980477..0cbc90e 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCheckWebUI.py
@@ -24,6 +24,8 @@ import sys
from mock.mock import MagicMock, patch
+# Needed to import checkWebUI.py
+sys.path.append("../../../../ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files")
import checkWebUI
class TestMain(unittest.TestCase):
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/test/python/ambari_agent/TestHardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
index 5c014db..038b2f8 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
@@ -52,7 +52,8 @@ class TestHardware(TestCase):
def test_build(self, get_os_version_mock, get_os_type_mock):
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
- hardware = Hardware()
+ config = None
+ hardware = Hardware(config)
result = hardware.get()
osdisks = hardware.osdisks()
for dev_item in result['mounts']:
@@ -196,7 +197,8 @@ class TestHardware(TestCase):
hostname_mock.return_value = 'ambari'
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
- result = Facter().facterInfo()
+ config = None
+ result = Facter(config).facterInfo()
self.assertEquals(result['hostname'], "ambari")
self.assertEquals(result['domain'], "apache.org")
@@ -211,7 +213,8 @@ class TestHardware(TestCase):
facter_setDataUpTimeOutput_mock.return_value = "262813.00 123.45"
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
- result = Facter().facterInfo()
+ config = None
+ result = Facter(config).facterInfo()
self.assertEquals(result['uptime_seconds'], '262813')
self.assertEquals(result['uptime_hours'], '73')
@@ -236,7 +239,8 @@ SwapFree: 1598676 kB
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
- result = Facter().facterInfo()
+ config = None
+ result = Facter(config).facterInfo()
self.assertEquals(result['memorysize'], 1832392)
self.assertEquals(result['memorytotal'], 1832392)
@@ -261,7 +265,8 @@ SwapFree: 1598676 kB
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
- result = Facter().facterInfo()
+ config = None
+ result = Facter(config).facterInfo()
self.assertTrue(inet_ntoa_mock.called)
self.assertTrue(get_ip_address_by_ifname_mock.called)
@@ -287,7 +292,8 @@ SwapFree: 1598676 kB
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
- result = Facter().facterInfo()
+ config = None
+ result = Facter(config).facterInfo()
self.assertTrue(get_ip_address_by_ifname_mock.called)
self.assertEquals(result['netmask'], None)
@@ -301,22 +307,23 @@ SwapFree: 1598676 kB
get_os_version_mock.return_value = "11"
get_os_family_mock.return_value = "redhat"
- result = Facter().facterInfo()
+ config = None
+ result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'redhat')
get_os_family_mock.return_value = "ubuntu"
- result = Facter().facterInfo()
+ result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'ubuntu')
get_os_family_mock.return_value = "suse"
- result = Facter().facterInfo()
+ result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'suse')
get_os_family_mock.return_value = "My_new_family"
- result = Facter().facterInfo()
+ result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'My_new_family')
@@ -352,8 +359,7 @@ SwapFree: 1598676 kB
json_data.items.return_value = [('key', 'value')]
json_data.__getitem__.return_value = 'value'
- facter = Facter()
- facter.config = config
+ facter = Facter(config)
result = facter.getSystemResourceOverrides()
isdir.assert_called_with('/etc/custom_resource_overrides')
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e96cf51/ambari-agent/src/test/python/ambari_agent/TestMain.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestMain.py b/ambari-agent/src/test/python/ambari_agent/TestMain.py
index ffb5d83..6f38410 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestMain.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestMain.py
@@ -96,10 +96,12 @@ class TestMain(unittest.TestCase):
setLevel_mock.assert_called_with(logging.DEBUG)
+ @patch("os.path.exists")
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
@patch.object(main.logger, "setLevel")
@patch("logging.basicConfig")
- def test_update_log_level(self, basicConfig_mock, setLevel_mock):
+ def test_update_log_level(self, basicConfig_mock, setLevel_mock, os_path_exists_mock):
+ os_path_exists_mock.return_value = False
config = AmbariConfig().getConfig()
# Testing with default setup (config file does not contain loglevel entry)