You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/03/27 23:45:53 UTC
svn commit: r1461886 - in /incubator/ambari/trunk/ambari-agent: conf/unix/
src/main/python/ambari_agent/
Author: swagle
Date: Wed Mar 27 22:45:53 2013
New Revision: 1461886
URL: http://svn.apache.org/r1461886
Log:
AMBARI-1728. Cleanup INFO Logging at the ambari agent to make it more useful and less verbose. (swagle)
Modified:
incubator/ambari/trunk/ambari-agent/conf/unix/ambari-agent.ini
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Controller.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/NetUtil.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/main.py
incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
Modified: incubator/ambari/trunk/ambari-agent/conf/unix/ambari-agent.ini
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/conf/unix/ambari-agent.ini?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/conf/unix/ambari-agent.ini (original)
+++ incubator/ambari/trunk/ambari-agent/conf/unix/ambari-agent.ini Wed Mar 27 22:45:53 2013
@@ -19,6 +19,8 @@ secured_url_port=8441
[agent]
prefix=/var/lib/ambari-agent/data
+;loglevel=(DEBUG/INFO)
+loglevel=INFO
[stack]
installprefix=/var/ambari-agent/
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/ActionQueue.py Wed Mar 27 22:45:53 2013
@@ -81,7 +81,10 @@ class ActionQueue(threading.Thread):
return self._stop.isSet()
def put(self, command):
- logger.info("The " + command['commandType'] + " from the server is \n" + pprint.pformat(command))
+ logger.info("Adding " + command['commandType'] + " for service " +\
+ command['serviceName'] + " of cluster " +\
+ command['clusterName'] + " to the queue.")
+ logger.debug(pprint.pformat(command))
self.commandQueue.put(command)
pass
@@ -90,7 +93,7 @@ class ActionQueue(threading.Thread):
while not self.stopped():
while not self.commandQueue.empty():
command = self.commandQueue.get()
- logger.info("Took an element of Queue: " + pprint.pformat(command))
+ logger.debug("Took an element of Queue: " + pprint.pformat(command))
if command['commandType'] == self.EXECUTION_COMMAND:
try:
#pass a copy of action since we don't want anything to change in the
@@ -118,8 +121,10 @@ class ActionQueue(threading.Thread):
livestatus = LiveStatus(cluster, service, component,
globalConfig, self.config)
result = livestatus.build()
- logger.info("Got live status for component " + component + " of service " + str(service) +\
- " of cluster " + str(cluster) + "\n" + pprint.pformat(result))
+ logger.debug("Got live status for component " + component +\
+ " of service " + str(service) +\
+ " of cluster " + str(cluster))
+ logger.debug(pprint.pformat(result))
if result is not None:
self.resultQueue.put((ActionQueue.STATUS_COMMAND, result))
except Exception, err:
@@ -172,7 +177,6 @@ class ActionQueue(threading.Thread):
return result
def executeCommand(self, command):
- logger.info("Executing command \n" + pprint.pformat(command))
clusterName = command['clusterName']
commandId = command['commandId']
hostname = command['hostname']
@@ -183,6 +187,11 @@ class ActionQueue(threading.Thread):
configurations = command['configurations']
result = []
+ logger.info("Executing command with id = " + str(commandId) +\
+ " for role = " + command['role'] + " of " +\
+ "cluster " + clusterName)
+ logger.debug(pprint.pformat(command))
+
taskId = command['taskId']
# Preparing 'IN_PROGRESS' report
self.commandInProgress = {
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Controller.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Controller.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Controller.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Controller.py Wed Mar 27 22:45:53 2013
@@ -113,11 +113,11 @@ class Controller(threading.Thread):
""" Put the required actions into the Queue """
""" Verify if the action is to reboot or not """
if not commands:
- logger.info("No commands from the server : " + pprint.pformat(commands))
+ logger.debug("No commands from the server : " + pprint.pformat(commands))
else:
"""Only add to the queue if not empty list """
for command in commands:
- logger.info("Adding command to the action queue: \n" +
+ logger.debug("Adding command to the action queue: \n" +\
pprint.pformat(command))
self.actionQueue.put(command)
pass
@@ -150,7 +150,7 @@ class Controller(threading.Thread):
response = self.sendRequest(self.heartbeatUrl, data)
response = json.loads(response)
- logger.info('Got server response: ' + pprint.pformat(response))
+ logger.debug('Got server response: ' + pprint.pformat(response))
serverId=int(response['responseId'])
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Heartbeat.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Heartbeat.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Heartbeat.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/Heartbeat.py Wed Mar 27 22:45:53 2013
@@ -54,18 +54,21 @@ class Heartbeat:
'hostname' : hostname.hostname(),
'nodeStatus' : nodeStatus
}
+
+ if len(queueResult) != 0:
+ heartbeat['reports'] = queueResult['reports']
+ heartbeat['componentStatus'] = queueResult['componentStatus']
+ pass
+ logger.info("Heartbeat : " + pformat(heartbeat))
+
if (int(id) >= 0) and state_interval > 0 and (int(id) % state_interval) == 0:
hostInfo = HostInfo()
nodeInfo = { }
# for now, just do the same work as registration
hostInfo.register(nodeInfo)
heartbeat['agentEnv'] = nodeInfo
+ logger.debug("agentEnv : " + str(nodeInfo))
- if len(queueResult) != 0:
- heartbeat['reports'] = queueResult['reports']
- heartbeat['componentStatus'] = queueResult['componentStatus']
- pass
- logger.info("Heartbeat dump: " + pformat(heartbeat))
return heartbeat
def main(argv=None):
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/LiveStatus.py Wed Mar 27 22:45:53 2013
@@ -125,8 +125,8 @@ class LiveStatus:
read_stack_version(component["componentName"])
}
break
- logger.info("The live status for component " + str(self.component) + " of service " + \
- str(self.service) + " is " + str(livestatus))
+ logger.debug("The live status for component " + str(self.component) +\
+ " of service " + str(self.service) + " is " + str(livestatus))
return livestatus
def main(argv=None):
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/NetUtil.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/NetUtil.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/NetUtil.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/NetUtil.py Wed Mar 27 22:45:53 2013
@@ -41,14 +41,14 @@ class NetUtil:
"""Try to connect to a given url. Result is True if url returns HTTP code 200, in any other case
(like unreachable server or wrong HTTP code) result will be False
"""
- logger.info("DEBUG:: Connecting to the following url " + url);
+ logger.info("Connecting to the following url " + url);
try:
parsedurl = urlparse(url)
ca_connection = httplib.HTTPSConnection(parsedurl[1])
ca_connection.request("GET", parsedurl[2])
response = ca_connection.getresponse()
status = response.status
- logger.info("DEBUG: Calling url received " + str(status))
+ logger.info("Calling url received " + str(status))
if status == 200:
return True
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py Wed Mar 27 22:45:53 2013
@@ -157,7 +157,7 @@ class PuppetExecutor:
rubyLib = ""
if os.environ.has_key("RUBYLIB"):
rubyLib = os.environ["RUBYLIB"]
- logger.info("RUBYLIB from Env " + rubyLib)
+ logger.debug("RUBYLIB from Env " + rubyLib)
if not (self.facterLib() in rubyLib):
rubyLib = rubyLib + ":" + self.facterLib()
if not (self.puppetLib() in rubyLib):
@@ -166,10 +166,10 @@ class PuppetExecutor:
tmperr = open(tmperrfile, 'w')
puppetEnv["RUBYLIB"] = rubyLib
puppetEnv = self.configureEnviron(puppetEnv)
- logger.info("Setting RUBYLIB as: " + rubyLib)
+ logger.debug("Setting RUBYLIB as: " + rubyLib)
logger.info("Running command " + pprint.pformat(puppetcommand))
puppet = self.lauch_puppet_subprocess(puppetcommand,tmpout, tmperr, puppetEnv)
- logger.info("Launching watchdog thread")
+ logger.debug("Launching watchdog thread")
self.event.clear()
self.last_puppet_has_been_killed = False
thread = Thread(target = self.puppet_watchdog_func, args = (puppet, ))
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py Wed Mar 27 22:45:53 2013
@@ -53,7 +53,7 @@ class PythonExecutor:
pythonCommand = self.pythonCommand(file)
logger.info("Running command " + pprint.pformat(pythonCommand))
process = self.lauch_python_subprocess(pythonCommand, tmpout, tmperr)
- logger.info("Launching watchdog thread")
+ logger.debug("Launching watchdog thread")
self.event.clear()
self.python_process_has_been_killed = False
thread = Thread(target = self.python_watchdog_func, args = (process, ))
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/StatusCheck.py Wed Mar 27 22:45:53 2013
@@ -102,7 +102,7 @@ class StatusCheck:
try:
pidPath = None
pidPattern = self.serToPidDict[serviceCode]
- logger.info('pidPattern: ' + pidPattern)
+ logger.debug('pidPattern: ' + pidPattern)
except KeyError as e:
logger.warn('There is no mapping for ' + serviceCode)
return None
@@ -110,7 +110,7 @@ class StatusCheck:
for pidFile in self.pidFilesDict.keys():
if re.match(pidPattern, pidFile):
pidPath = self.pidFilesDict[pidFile]
- logger.info('pidPath: ' + str(pidPath))
+ logger.debug('pidPath: ' + str(pidPath))
result = self.getIsLive(pidPath)
return result
except KeyError:
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/main.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/main.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/main.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/main.py Wed Mar 27 22:45:53 2013
@@ -80,17 +80,18 @@ def main():
if options.verbose:
logging.basicConfig(format=formatstr, level=logging.DEBUG, filename=logfile)
logger.setLevel(logging.DEBUG)
+ logger.info("loglevel=logging.DEBUG")
else:
logging.basicConfig(format=formatstr, level=logging.INFO, filename=logfile)
logger.setLevel(logging.INFO)
-
- logger.debug("loglevel=logging.DEBUG")
+ logger.info("loglevel=logging.INFO")
default_cfg = { 'agent' : { 'prefix' : '/home/ambari' } }
config = ConfigParser.RawConfigParser(default_cfg)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGUSR1, debug)
+
if (len(sys.argv) >1) and sys.argv[1]=='stop':
# stop existing Ambari agent
pid = -1
@@ -143,6 +144,21 @@ def main():
killstaleprocesses()
+ # Setting loglevel based on config file
+ try:
+ loglevel = config.get('agent', 'loglevel')
+ if loglevel is not None:
+ if loglevel == 'DEBUG':
+ logging.basicConfig(format=formatstr, level=logging.DEBUG, filename=logfile)
+ logger.setLevel(logging.DEBUG)
+ logger.info("Newloglevel=logging.DEBUG")
+ else:
+ logging.basicConfig(format=formatstr, level=logging.INFO, filename=logfile)
+ logger.setLevel(logging.INFO)
+ logger.debug("Newloglevel=logging.INFO")
+ except Exception, err:
+ logger.info("Default loglevel=DEBUG")
+
server_url = 'https://' + config.get('server', 'hostname') + ':' + config.get('server', 'url_port')
print("Connecting to the server at " + server_url + "...")
logger.info('Connecting to the server at: ' + server_url)
Modified: incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py?rev=1461886&r1=1461885&r2=1461886&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py Wed Mar 27 22:45:53 2013
@@ -25,7 +25,7 @@ from uuid import getnode as get_mac
from shell import shellRunner
from datetime import datetime
import AmbariConfig
-
+import pprint
logger = logging.getLogger()
@@ -47,8 +47,8 @@ def writeImports(outputFile, modulesdir,
def generateManifest(parsedJson, fileName, modulesdir, ambariconfig):
- logger.info("JSON Received:")
- logger.info(json.dumps(parsedJson, sort_keys=True, indent=4))
+ logger.debug("JSON Received:")
+ logger.debug(json.dumps(parsedJson, sort_keys=True, indent=4))
#reading json
hostname = parsedJson['hostname']
clusterHostInfo = {}
@@ -164,6 +164,7 @@ def writeHostAttributes(outputFile, host
#write flat configurations
def writeFlatConfigurations(outputFile, flatConfigs):
flatDict = {}
+ logger.info("Generating global configurations =>\n" + pprint.pformat(flatConfigs))
for flatConfigName in flatConfigs.iterkeys():
for flatConfig in flatConfigs[flatConfigName].iterkeys():
flatDict[flatConfig] = flatConfigs[flatConfigName][flatConfig]
@@ -175,9 +176,8 @@ def writeNonGlobalConfigurations(outputF
outputFile.write('$configuration = {\n')
for configName in xmlConfigs.iterkeys():
-
config = xmlConfigs[configName]
-
+ logger.info("Generating " + configName + ",configurations =>\n" + pprint.pformat(config))
outputFile.write(configName + '=> {\n')
coma = ''
for configParam in config.iterkeys():
@@ -235,6 +235,7 @@ def writeTasks(outputFile, roles, ambari
stageNum = stageNum + 1
outputFile.write('}\n')
+
def normalizeTaskParams(taskParams):
result = ''
coma = ''