You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2008/03/03 18:01:15 UTC
svn commit: r633173 - in /hadoop/core/branches/branch-0.16: ./
src/contrib/hod/bin/ src/contrib/hod/hodlib/Common/
Author: nigel
Date: Mon Mar 3 09:01:12 2008
New Revision: 633173
URL: http://svn.apache.org/viewvc?rev=633173&view=rev
Log:
Merge of -r 633170:633171 from trunk to branch 0.16 to fix HADOOP-2809.
Modified:
hadoop/core/branches/branch-0.16/CHANGES.txt
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring
hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster
hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/logger.py
hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/types.py
Modified: hadoop/core/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/CHANGES.txt?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/CHANGES.txt Mon Mar 3 09:01:12 2008
@@ -94,6 +94,9 @@
HADOOP-2847. Ensure idle cluster cleanup works even if the JobTracker
becomes unresponsive to RPC calls. (Hemanth Yamijala via nigel)
+ HADOOP-2809. Fix HOD syslog config syslog-address so that it works.
+ (Hemanth Yamijala via nigel)
+
Release 0.16.0 - 2008-02-07
INCOMPATIBLE CHANGES
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hod Mon Mar 3 09:01:12 2008
@@ -182,7 +182,7 @@
False, os.path.join(rootDirectory, 'logs'), False, False),
('syslog-address', 'address', 'Syslog address.',
- False, None, False, False),
+ False, None, False, True),
('xrs-port-range', 'range', 'XML-RPC port range n-m.',
False, None, True, True),
@@ -292,7 +292,7 @@
False, None, False, False),
('syslog-address', 'address', 'Syslog address.',
- False, None, False, False),
+ False, None, False, True),
('java-home', 'directory', 'Java home directory.',
False, None, True, False),
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodcleanup Mon Mar 3 09:01:12 2008
@@ -50,6 +50,7 @@
from hodlib.Common.util import local_fqdn, tar, filter_warnings,\
get_exception_string, get_exception_error_string
from hodlib.Common.logger import hodLog
+from hodlib.Common.logger import getLogger
filter_warnings()
@@ -62,51 +63,6 @@
VERSION = vFile.readline()
vFile.close()
-def getLogger(hodRingOptions):
- # print >>sys.stderr,"now"
- if hodRingOptions['hodring-debug'] > 0:
- _baseLogger = hodLog('hodring')
- log = _baseLogger.add_logger('main')
-
- if hodRingOptions.has_key('hodring-stream'):
- if hodRingOptions['hodring-stream']:
- _baseLogger.add_stream(level=hodRingOptions['hodring-debug'],
- addToLoggerNames=('main',))
-
- _serviceID = hodRingOptions['service-id']
- if hodRingOptions['hodring-log-dir']:
- if _serviceID:
- __logDir = os.path.join(hodRingOptions['hodring-log-dir'], "%s.%s" % (
- hodRingOptions['user-id'], _serviceID))
- else:
- __logDir = os.path.join(hodRingOptions['hodring-log-dir'],
- hodRingOptions['user-id'])
- if not os.path.exists(__logDir):
- os.mkdir(__logDir)
-
- _baseLogger.add_file(logDirectory=__logDir,
- level=hodRingOptions['hodring-debug'], addToLoggerNames=('main',))
-
- if hodRingOptions['hodring-syslog-address']:
- _baseLogger.add_syslog(hodRingOptions['hodring-syslog-address'],
- level=hodRingOptions['hodring-debug'], addToLoggerNames=('main',))
- return log
-
-def ensureLogDir(logDir):
- """Verify that the passed in log directory exists, and if it doesn't
- create it."""
- if not os.path.exists(logDir):
- try:
- old_mask = os.umask(0)
- os.makedirs(logDir, 01777)
- os.umask(old_mask)
- except Exception, e:
- print >>sys.stderr, "Could not create log directories %s. Exception: \
- %s. Stack Trace: %s" % (logDir, get_exception_error_string()
- , get_exception_string())
- raise e
-
-
def __archive_logs(conf, log):
# need log-destination-uri, __hadoopLogDirs, temp-dir
status = True
@@ -206,19 +162,28 @@
action="store_true")
(options, args) = parser.parse_args()
_options= {}
+ _options['hodring'] = {}
for opt in dir(options):
if "--"+opt in option_list:
_options[opt] = getattr(options,opt)
if _options.has_key('hadoop-log-dirs') and _options['hadoop-log-dirs']:
_options['hadoop-log-dirs'] = _options['hadoop-log-dirs'].split(",")
- _options['hodring-debug'] = int(_options['hodring-debug'])
+ if _options.has_key('hodring-syslog-address') and _options['hodring-syslog-address']:
+ _options['hodring']['syslog-address'] = \
+ _options['hodring-syslog-address'].split(':')
+ _options['hodring']['debug'] = int(_options['hodring-debug'])
+ _options['hodring']['log-dir'] = _options['hodring-log-dir']
+ _options['hodring']['stream'] = _options['hodring-stream']
+ _options['hodring']['userid'] = _options['user-id']
+ os.putenv('PBS_JOBID', _options['service-id'] )
return _options
-
if __name__ == '__main__':
+ log = None
try:
conf = unpack()
- log = getLogger(conf,)
+ # Use the same log as hodring
+ log = getLogger(conf['hodring'],'hodring')
log.debug("Logger initialised successfully")
status = __archive_logs(conf,log)
log.info("Archive status : %s" % status)
@@ -232,4 +197,4 @@
log.info("Cleanup successfully completed")
except Exception, e:
if log:
- log.info("Stack trace:\n%s" %(get_exception_error_string(),get_exception_string()))
+ log.info("Stack trace:\n%s\n%s" %(get_exception_error_string(),get_exception_string()))
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/hodring Mon Mar 3 09:01:12 2008
@@ -47,7 +47,7 @@
from hodlib.HodRing.hodRing import HodRing
from hodlib.Common.setup import *
from hodlib.Common.util import filter_warnings,get_exception_string, get_exception_error_string
-from hodlib.Common.logger import hodLog
+from hodlib.Common.logger import getLogger, ensureLogDir
filter_warnings()
@@ -133,48 +133,6 @@
False, 2, False, True))
}
-def getLogger(hodRingOptions):
- if hodRingOptions['hodring']['debug'] > 0:
- _baseLogger = hodLog('hodring')
- log = _baseLogger.add_logger('main')
-
- if hodRingOptions['hodring'].has_key('stream'):
- if hodRingOptions['hodring']['stream']:
- _baseLogger.add_stream(level=hodRingOptions['hodring']['debug'],
- addToLoggerNames=('main',))
-
- _serviceID = os.getenv('PBS_JOBID')
-
- if hodRingOptions['hodring'].has_key('log-dir'):
- if _serviceID:
- __logDir = os.path.join(hodRingOptions['hodring']['log-dir'], "%s.%s" % (
- hodRingOptions['hodring']['userid'], _serviceID))
- else:
- __logDir = os.path.join(hodRingOptions['hodring']['log-dir'],
- hodRingOptions['hodring']['userid'])
- if not os.path.exists(__logDir):
- os.mkdir(__logDir)
-
- _baseLogger.add_file(logDirectory=__logDir,
- level=hodRingOptions['hodring']['debug'], addToLoggerNames=('main',))
-
- if hodRingOptions['hodring'].has_key('syslog-address'):
- _baseLogger.add_syslog(hodRingOptions['hodring']['syslog-address'],
- level=hodRingOptions['hodring']['debug'], addToLoggerNames=('main',))
- return log
-
-def ensureLogDir(logDir):
- """Verify that the passed in log directory exists, and if it doesn't
- create it."""
- if not os.path.exists(logDir):
- try:
- old_mask = os.umask(0)
- os.makedirs(logDir, 01777)
- os.umask(old_mask)
- except Exception, e:
- print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
- raise e
-
if __name__ == '__main__':
confDef = definition()
@@ -194,7 +152,7 @@
if service.log:
log = service.log
else:
- log = getLogger(hodRingOptions)
+ log = getLogger(hodRingOptions['hodring'],'hodring')
list = []
@@ -244,8 +202,9 @@
+ ",".join(list)
if hodRingOptions['hodring'].has_key('syslog-address'):
- cmdString = cmdString + " --hodring-syslog-address " \
- + hodRingOptions['hodring']['syslog-address']
+ syslogAddr = hodRingOptions['hodring']['syslog-address'][0] + \
+ ':' + str(hodRingOptions['hodring']['syslog-address'][1])
+ cmdString = cmdString + " --hodring-syslog-address " + syslogAddr
if service._cfg.has_key('pkgs'):
cmdString = cmdString + " --pkgs " + service._cfg['pkgs']
@@ -292,5 +251,5 @@
if service.log:
log = service.log
else:
- log = getLogger(hodRingOptions)
+ log = getLogger(hodRingOptions['hodring'], 'hodring')
log.error("Error in bin/hodring %s. \nStack trace:\n%s" %(get_exception_error_string(),get_exception_string()))
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/bin/ringmaster Mon Mar 3 09:01:12 2008
@@ -47,7 +47,7 @@
from hodlib.Common.setup import *
from hodlib.Common.descGenerator import *
from hodlib.Common.util import local_fqdn, filter_warnings
-from hodlib.Common.logger import hodLog
+from hodlib.Common.logger import getLogger, ensureLogDir
from hodlib.Common.util import get_exception_string, get_exception_error_string
filter_warnings()
@@ -269,59 +269,21 @@
defOrder = [ 'ringmaster', 'hodring', 'resource_manager',
'gridservice-mapred', 'gridservice-hdfs' ]
-def getLogger(cfg):
- user = cfg['userid']
- baseLogger = hodLog('ringmaster')
- log = baseLogger.add_logger('main')
-
- if cfg['stream']:
- baseLogger.add_stream(level=cfg['debug'], addToLoggerNames=('main',))
-
- if cfg.has_key('log-dir'):
- serviceId = os.getenv('PBS_JOBID')
- if serviceId:
- logDir = os.path.join(cfg['log-dir'], "%s.%s" % (user, serviceId))
- else:
- logDir = os.path.join(cfg['log-dir'], user)
- if not os.path.exists(logDir):
- os.mkdir(logDir)
-
- baseLogger.add_file(logDirectory=logDir, level=cfg['debug'],
- addToLoggerNames=('main',))
-
- if cfg.has_key('syslog-address'):
- baseLogger.add_syslog(cfg['syslog-address'],
- level=cfg['debug'], addToLoggerNames=('main',))
-
- return log
-
-def ensureLogDir(logDir):
- """Verify that the passed in log directory exists, and if it doesn't
- create it."""
- if not os.path.exists(logDir):
- try:
- old_mask = os.umask(0)
- os.makedirs(logDir, 01777)
- os.umask(old_mask)
- except Exception, e:
- print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
- raise e
-
-
if __name__ == '__main__':
confDef = definition()
confDef.add_defs(defList, defOrder)
ringMasterOptions = options(confDef, "./%s [OPTIONS]" % myName, VERSION)
- ensureLogDir(ringMasterOptions['ringmaster']['log-dir'])
log = None
try:
- log = getLogger(ringMasterOptions['ringmaster'])
(status, statusMsgs) = ringMasterOptions.verify()
if not status:
raise Exception("%s" % statusMsgs)
ringMasterOptions.replace_escape_seqs()
ringMasterOptions['ringmaster']['base-dir'] = rootDirectory
+
+ ensureLogDir(ringMasterOptions['ringmaster']['log-dir'])
+ log = getLogger(ringMasterOptions['ringmaster'],'ringmaster')
ret = main(ringMasterOptions,log)
sys.exit(ret)
except Exception, e:
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/logger.py
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/logger.py?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/logger.py (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/logger.py Mon Mar 3 09:01:12 2008
@@ -376,7 +376,7 @@
def add_syslog_handler(loggerName):
if not self.__logObjs['syslog'].has_key(loggerName):
loggingLevel = self.__get_logging_level(level,
- defaultStreamLevel)
+ defaultSyslogLevel)
address[1] = int(address[1])
syslogHandler = logging.handlers.SysLogHandler(tuple(address),
@@ -738,3 +738,45 @@
pass
+def ensureLogDir(logDir):
+ """Verify that the passed in log directory exists, and if it doesn't
+ create it."""
+ if not os.path.exists(logDir):
+ try:
+ old_mask = os.umask(0)
+ os.makedirs(logDir, 01777)
+ os.umask(old_mask)
+ except Exception, e:
+ print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
+ raise e
+
+def getLogger(cfg, logName):
+ if cfg['debug'] > 0:
+ user = cfg['userid']
+ baseLogger = hodLog(logName)
+ log = baseLogger.add_logger('main')
+
+ if cfg.has_key('log-dir'):
+ serviceId = os.getenv('PBS_JOBID')
+ if serviceId:
+ logDir = os.path.join(cfg['log-dir'], "%s.%s" % (user, serviceId))
+ else:
+ logDir = os.path.join(cfg['log-dir'], user)
+ if not os.path.exists(logDir):
+ os.mkdir(logDir)
+
+ baseLogger.add_file(logDirectory=logDir, level=cfg['debug'],
+ addToLoggerNames=('main',))
+
+ try:
+ if cfg.has_key('stream') and cfg['stream']:
+ baseLogger.add_stream(level=cfg['debug'], addToLoggerNames=('main',))
+
+ if cfg.has_key('syslog-address'):
+ baseLogger.add_syslog(cfg['syslog-address'],
+ level=cfg['debug'], addToLoggerNames=('main',))
+ except Exception,e:
+ # Caught an exception while initialising logger
+ log.critical("%s Logger failed to initialise. Reason : %s" % (logName, e))
+ pass
+ return log
Modified: hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/types.py
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/types.py?rev=633173&r1=633172&r2=633173&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/types.py (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hod/hodlib/Common/types.py Mon Mar 3 09:01:12 2008
@@ -477,11 +477,14 @@
def __verify_address(self, type, value):
valueInfo = self.__get_value_info()
- socket = tcpSocket(value)
- if socket.verify():
- self.__set_value_info(valueInfo, normalized=self.normalize(type,
+ try:
+ socket = tcpSocket(value)
+ if socket.verify():
+ self.__set_value_info(valueInfo, normalized=self.normalize(type,
value))
- else:
+ else:
+ self.__set_value_info(valueInfo)
+ except:
self.__set_value_info(valueInfo)
return valueInfo