You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2008/01/04 19:20:22 UTC
svn commit: r608950 [2/7] - in /lucene/hadoop/trunk/src: contrib/hod/
contrib/hod/bin/ contrib/hod/conf/ contrib/hod/hodlib/
contrib/hod/hodlib/AllocationManagers/ contrib/hod/hodlib/Common/
contrib/hod/hodlib/GridServices/ contrib/hod/hodlib/Hod/ cont...
Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py Fri Jan 4 10:20:17 2008
@@ -0,0 +1,335 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+"""manage component descriptors"""
+# -*- python -*-
+
+import random
+
+from sets import Set
+from pprint import pformat
+from hodlib.Common.util import local_fqdn
+from hodlib.Common.tcp import tcpSocket, tcpError
+
+class Schema:
+ """the primary class for describing
+ schema's """
+ STRING, LIST, MAP = range(3)
+
+ def __init__(self, name, type = STRING, delim=','):
+ self.name = name
+ self.type = type
+ self.delim = delim
+
+ def getName(self):
+ return self.name
+
+ def getType(self):
+ return self.type
+
+ def getDelim(self):
+ return self.delim
+
+class _Merger:
+ """A class to merge lists and add key/value
+ pairs to a dictionary"""
+ def mergeList(x, y, uniq=True):
+ l = []
+ l.extend(x)
+ l.extend(y)
+ if not uniq:
+ return l
+
+ s = Set(l)
+ l = list(s)
+ return l
+
+ mergeList = staticmethod(mergeList)
+
+ def mergeMap(to, add):
+
+ for k in add:
+ to.setdefault(k, add[k])
+
+ return to
+
+ mergeMap = staticmethod(mergeMap)
+
+class NodePoolDesc:
+ """a schema for describing
+ Nodepools"""
+ def __init__(self, dict):
+ self.dict = dict.copy()
+
+ self.dict.setdefault('attrs', {})
+
+ self._checkRequired()
+
+ def _checkRequired(self):
+
+ if not 'id' in self.dict:
+ raise ValueError, "nodepool needs 'id'"
+
+ if self.getPkgDir() == None:
+ raise ValueError, "nodepool %s needs 'pkgs'" % (self.getName())
+
+ def getName(self):
+ return self.dict['id']
+
+ def getPkgDir(self):
+ return self.dict['batch-home']
+
+ def getAttrs(self):
+ return self.dict['attrs']
+
+ def getSchema():
+ schema = {}
+
+ s = Schema('id')
+ schema[s.getName()] = s
+
+ s = Schema('batch-home', Schema.LIST, ':')
+ schema[s.getName()] = s
+
+ s = Schema('attrs', Schema.MAP)
+ schema[s.getName()] = s
+
+ return schema
+
+ getSchema = staticmethod(getSchema)
+
+class ServiceDesc:
+ """A schema for describing services"""
+ def __init__(self, dict):
+ self.dict = dict.copy()
+
+ self.dict.setdefault('external', False)
+ self.dict.setdefault('attrs', {})
+ self.dict.setdefault('java-opts', [])
+ self.dict.setdefault('envs', {})
+ self.dict.setdefault('host',None)
+ self.dict.setdefault('port',None)
+ self.dict.setdefault('tar', None)
+ self.dict.setdefault('pkgs', '')
+ self.dict.setdefault('final-attrs', {})
+ self._checkRequired()
+ self.__dict_update()
+
+ def __dict_update(self):
+ getattr(self, "_%s" % self.dict['id'])()
+
+ def _mapred(self):
+ if self.isExternal():
+ self.dict['final-attrs']['mapred.job.tracker'] = "%s:%s" % (self.dict['host'],
+ self.dict['tracker_port'])
+
+ # self.dict['final-attrs']['mapred.job.tracker.info.port'] = \
+ # str(self.dict['info_port'])
+ # After Hadoop-2185
+ self.dict['final-attrs']['mapred.job.tracker.http.bindAddress'] = \
+ "%s:%s" %(self.dict['host'], self.dict['info_port'])
+
+ if self.dict.has_key('hadoop-tar-ball'):
+ self.dict['tar'] = self.dict['hadoop-tar-ball']
+
+ def _hdfs(self):
+ if self.isExternal():
+ self.dict['final-attrs']['fs.default.name'] = "%s:%s" % (self.dict['host'],
+ self.dict['fs_port'])
+
+ # self.dict['final-attrs']['dfs.info.port'] = str(self.dict['info_port'])
+ # After Hadoop-2185
+ self.dict['final-attrs']['dfs.http.bindAddress'] = "%s:%s" % \
+ (self.dict['host'], self.dict['info_port'])
+
+ if self.dict.has_key('hadoop-tar-ball'):
+ self.dict['tar'] = self.dict['hadoop-tar-ball']
+
+ def _checkRequired(self):
+
+ if not 'id' in self.dict:
+ raise ValueError, "service description needs 'id'"
+
+# if len(self.getPkgDirs()) <= 0:
+# raise ValueError, "service description %s needs 'pkgs'" % (self.getName())
+
+ def getName(self):
+ return self.dict['id']
+
+ def isExternal(self):
+ """True if the service is outside hod.
+ e.g. connect to existing HDFS"""
+
+ return self.dict['external']
+
+ def getPkgDirs(self):
+ return self.dict['pkgs']
+
+ def getTar(self):
+ return self.dict['tar']
+
+ def getAttrs(self):
+ return self.dict['attrs']
+
+ def getfinalAttrs(self):
+ return self.dict['final-attrs']
+
+ def getJavaOpts(self):
+ return self.dict['java-opts']
+
+ def getEnvs(self):
+ return self.dict['envs']
+
+ def getSchema():
+ schema = {}
+
+ s = Schema('id')
+ schema[s.getName()] = s
+
+ s = Schema('external')
+ schema[s.getName()] = s
+
+ s = Schema('pkgs', Schema.LIST, ':')
+ schema[s.getName()] = s
+
+ s = Schema('tar', Schema.LIST, ":")
+ schema[s.getName()] = s
+
+ s = Schema('attrs', Schema.MAP)
+ schema[s.getName()] = s
+
+ s = Schema('final-attrs', Schema.MAP)
+ schema[s.getName()] = s
+
+ s = Schema('envs', Schema.MAP)
+ schema[s.getName()] = s
+
+ s = Schema('java-opts', Schema.MAP)
+ schema[s.getName()] = s
+
+ return schema
+
+ getSchema = staticmethod(getSchema)
+
+class CommandDesc:
+
+ def __init__(self, dict):
+ """a class for how a command is described"""
+ self.dict = dict
+
+ def __repr__(self):
+ return pformat(self.dict)
+
+ def _getName(self):
+ """return the name of the command to be run"""
+ return self.dict['name']
+
+ def _getProgram(self):
+ """return where the program is """
+ return self.dict['program']
+
+ def _getArgv(self):
+ """return the arguments for the command to be run"""
+ return self.dict['argv']
+
+ def _getEnvs(self):
+ """return the environment in which the command is to be run"""
+ return self.dict['envs']
+
+ def _getJavaOpts(self):
+ return self.dict['java-opts']
+
+ def _getPkgDirs(self):
+ """return the packages for this command"""
+ return self.dict['pkgdirs']
+
+ def _getWorkDirs(self):
+ """return the working directories for this command"""
+ return self.dict['workdirs']
+
+ def _getAttrs(self):
+ """return the list of attributes for this command"""
+ return self.dict['attrs']
+
+ def _getfinalAttrs(self):
+ """return the final xml params list for this command"""
+ return self.dict['final-attrs']
+
+ def _getForeground(self):
+ """return if the command is to be run in foreground or not"""
+ return self.dict['fg']
+
+ def _getStdin(self):
+ return self.dict['stdin']
+
+ def toString(cmdDesc):
+ """return a string representation of this command"""
+ row = []
+ row.append('name=%s' % (cmdDesc._getName()))
+ row.append('program=%s' % (cmdDesc._getProgram()))
+ row.append('pkgdirs=%s' % CommandDesc._csv(cmdDesc._getPkgDirs(), ':'))
+
+ if 'argv' in cmdDesc.dict:
+ row.append('argv=%s' % CommandDesc._csv(cmdDesc._getArgv()))
+
+ if 'envs' in cmdDesc.dict:
+ envs = cmdDesc._getEnvs()
+ list = []
+ for k in envs:
+ v = envs[k]
+ list.append('%s=%s' % (k, v))
+ row.append('envs=%s' % CommandDesc._csv(list))
+
+ if 'workdirs' in cmdDesc.dict:
+ row.append('workdirs=%s' % CommandDesc._csv(cmdDesc._getWorkDirs(), ':'))
+
+ if 'attrs' in cmdDesc.dict:
+ attrs = cmdDesc._getAttrs()
+ list = []
+ for k in attrs:
+ v = attrs[k]
+ list.append('%s=%s' % (k, v))
+ row.append('attrs=%s' % CommandDesc._csv(list))
+
+ if 'final-attrs' in cmdDesc.dict:
+ fattrs = cmdDesc._getAttrs()
+ list = []
+ for k in fattrs:
+ v = fattrs[k]
+ list.append('%s=%s' % (k, v))
+ row.append('final-attrs=%s' % CommandDesc._cvs(list))
+
+ if 'fg' in cmdDesc.dict:
+ row.append('fg=%s' % (cmdDesc._getForeground()))
+
+ if 'stdin' in cmdDesc.dict:
+ row.append('stdin=%s' % (cmdDesc._getStdin()))
+
+ return CommandDesc._csv(row)
+
+ toString = staticmethod(toString)
+
+ def _csv(row, delim=','):
+ """return a string in csv format"""
+ import cStringIO
+ import csv
+
+ queue = cStringIO.StringIO()
+ writer = csv.writer(queue, delimiter=delim, escapechar='\\', quoting=csv.QUOTE_NONE,
+ doublequote=False, lineterminator='\n')
+ writer.writerow(row)
+ return queue.getvalue().rstrip('\n')
+
+ _csv = staticmethod(_csv)
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py Fri Jan 4 10:20:17 2008
@@ -0,0 +1,72 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+"""manage hod configuration"""
+# -*- python -*-
+
+import sys, csv, os
+from optparse import Option, OptionParser
+from xml.dom import minidom
+from sets import Set
+from select import select, poll, POLLIN
+
+from hodlib.Common.desc import *
+
+class DescGenerator:
+ """Contains the conversion to descriptors and other method calls
+ to config"""
+ def __init__(self, hodConfig):
+ """parse all the descriptors"""
+
+ self.hodConfig = hodConfig
+
+ def initializeDesc(self):
+ self.hodConfig['nodepooldesc'] = self.createNodePoolDesc()
+ self.hodConfig['servicedesc'] = self.createServiceDescDict()
+
+ return self.hodConfig
+
+ def getServices(self):
+ """get all the services from the config"""
+
+ sdd = {}
+ for keys in self.hodConfig:
+ if keys.startswith('gridservice-'):
+ str = keys.split('-')
+ dict = self.hodConfig[keys]
+ if 'server-params' in dict: dict['attrs'] = dict['server-params']
+ if 'final-server-params' in dict: dict['final-attrs'] = dict['final-server-params']
+ dict['id'] = str[1]
+ desc = ServiceDesc(dict)
+ sdd[desc.getName()] = desc
+
+ return sdd
+
+ def createNodePoolDesc(self):
+ """ create a node pool descriptor and store
+ it in hodconfig"""
+
+ desc = NodePoolDesc(self.hodConfig['resource_manager'])
+ return desc
+
+ def createServiceDescDict(self):
+ """create a service descriptor for
+ all the services and store it in the
+ hodconfig"""
+
+ sdd = self.getServices()
+ return sdd
+
+
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py Fri Jan 4 10:20:17 2008
@@ -0,0 +1,229 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+# $Id:setup.py 5158 2007-04-09 00:14:35Z zim $
+#
+# Christopher Zimmerman - zim@yahoo-inc.com - 04/13/2007
+#------------------------------------------------------------------------------
+import os, time, shutil, xmlrpclib, socket, pprint
+
+from signal import *
+
+from hodlib.Common.logger import hodLog, hodDummyLogger
+from hodlib.Common.socketServers import hodXMLRPCServer
+from hodlib.Common.util import local_fqdn
+from hodlib.Common.xmlrpc import hodXRClient
+
+class hodBaseService:
+ """hodBaseService class - This class provides service registration, logging,
+ and configuration access methods. It also provides an XML-RPC server.
+ This class should be extended to create hod services. Methods beginning
+ with _xr_method will automatically be added to instances of this class.
+ """
+ def __init__(self, name, config, xrtype='threaded'):
+ """ Initialization requires a name string and a config object of type
+ hodlib.Common.setup.options or hodlib.Common.setup.config."""
+
+ self.name = name
+ self.hostname = local_fqdn()
+ self._cfg = config
+ self._xrc = None
+ self.logs = {}
+ self._baseLogger = None
+ self._serviceID = os.getenv('PBS_JOBID')
+
+ self.__logDir = None
+ self.__svcrgy = None
+ self.__stop = False
+ self.__xrtype = xrtype
+
+ self._init_logging()
+
+ self._init_signals()
+ self._init_xrc_server()
+
+ def __set_logging_level(self, level):
+ self.logs['main'].info("Setting log level to %s." % level)
+ for loggerName in self.loggers.keys():
+ self.logs['main'].set_logger_level(loggerName, level)
+
+ def __get_logging_level(self):
+ if self._cfg.has_key('stream'):
+ return self.loggers['main'].get_level('stream', 'main')
+ elif self._cfg.has_key('log-dir'):
+ return self.loggers['main'].get_level('file', 'main')
+ else:
+ return 0
+
+ def _xr_method_stop(self, *args):
+ """XML-RPC method, calls stop() on ourselves."""
+
+ return self.stop()
+
+ def _xr_method_status(self, *args):
+ """XML-RPC method, calls status() on ourselves."""
+
+ return self.status()
+
+ def _init_logging(self):
+ if self._cfg.has_key('debug'):
+ if self._cfg['debug'] > 0:
+ self._baseLogger = hodLog(self.name)
+ self.logs['main'] = self._baseLogger.add_logger('main')
+
+ if self._cfg.has_key('stream'):
+ if self._cfg['stream']:
+ self._baseLogger.add_stream(level=self._cfg['debug'],
+ addToLoggerNames=('main',))
+
+ if self._cfg.has_key('log-dir'):
+ if self._serviceID:
+ self.__logDir = os.path.join(self._cfg['log-dir'], "%s.%s" % (
+ self._cfg['userid'], self._serviceID))
+ else:
+ self.__logDir = os.path.join(self._cfg['log-dir'],
+ self._cfg['userid'])
+ if not os.path.exists(self.__logDir):
+ os.mkdir(self.__logDir)
+
+ self._baseLogger.add_file(logDirectory=self.__logDir,
+ level=self._cfg['debug'], addToLoggerNames=('main',))
+
+ if self._cfg.has_key('syslog-address'):
+ self._baseLogger.add_syslog(self._cfg['syslog-address'],
+ level=self._cfg['debug'], addToLoggerNames=('main',))
+
+ if not self.logs.has_key('main'):
+ self.logs['main'] = hodDummyLogger()
+ else:
+ self.logs['main'] = hodDummyLogger()
+ else:
+ self.logs['main'] = hodDummyLogger()
+
+ def _init_signals(self):
+ def sigStop(sigNum, handler):
+ self.sig_wrapper(sigNum, self.stop)
+
+ def toggleLevel():
+ currentLevel = self.__get_logging_level()
+ if currentLevel == 4:
+ self.__set_logging_level(1)
+ else:
+ self.__set_logging_level(currentLevel + 1)
+
+ def sigStop(sigNum, handler):
+ self._sig_wrapper(sigNum, self.stop)
+
+ def sigDebug(sigNum, handler):
+ self.sig_wrapper(sigNum, toggleLevel)
+
+ signal(SIGTERM, sigStop)
+ signal(SIGQUIT, sigStop)
+ signal(SIGINT, sigStop)
+ signal(SIGUSR2, sigDebug)
+
+ def _sig_wrapper(self, sigNum, handler, *args):
+ self.logs['main'].info("Caught signal %s." % sigNum)
+
+ if args:
+ handler(args)
+ else:
+ handler()
+
+ def _init_xrc_server(self):
+ host = None
+ ports = None
+ if self._cfg.has_key('xrs-address'):
+ (host, port) = (self._cfg['xrs-address'][0], self._cfg['xrs-address'][1])
+ ports = (port,)
+ elif self._cfg.has_key('xrs-port-range'):
+ host = ''
+ ports = self._cfg['xrs-port-range']
+
+ if host != None:
+ if self.__xrtype == 'threaded':
+ self._xrc = hodXMLRPCServer(host, ports)
+ elif self.__xrtype == 'twisted':
+ try:
+ from socketServers import twistedXMLRPCServer
+ self._xrc = twistedXMLRPCServer(host, ports, self.logs['main'])
+ except ImportError:
+ self.logs['main'].error("Twisted XML-RPC server not available, "
+ + "falling back on threaded server.")
+ self._xrc = hodXMLRPCServer(host, ports)
+ for attr in dir(self):
+ if attr.startswith('_xr_method_'):
+ self._xrc.register_function(getattr(self, attr),
+ attr[11:])
+
+ self._xrc.register_introspection_functions()
+
+ def _register_service(self, port=None, installSignalHandlers=1):
+ if self.__svcrgy:
+ self.logs['main'].info(
+ "Registering service with service registery %s... " % self.__svcrgy)
+ svcrgy = hodXRClient(self.__svcrgy, None, None, 0, 0, installSignalHandlers)
+
+ if self._xrc and self._http:
+ svcrgy.registerService(self._cfg['userid'], self._serviceID,
+ self.hostname, self.name, 'hod', {
+ 'xrs' : "http://%s:%s" % (
+ self._xrc.server_address[0],
+ self._xrc.server_address[1]),'http' :
+ "http://%s:%s" % (self._http.server_address[0],
+ self._http.server_address[1])})
+ elif self._xrc:
+ svcrgy.registerService(self._cfg['userid'], self._serviceID,
+ self.hostname, self.name, 'hod', {
+ 'xrs' : "http://%s:%s" % (
+ self._xrc.server_address[0],
+ self._xrc.server_address[1]),})
+ elif self._http:
+ svcrgy.registerService(self._cfg['userid'], self._serviceID,
+ self.hostname, self.name, 'hod', {'http' :
+ "http://%s:%s" % (self._http.server_address[0],
+ self._http.server_address[1]),})
+ else:
+ svcrgy.registerService(self._cfg['userid'], self._serviceID,
+ self.hostname, name, 'hod', {} )
+
+ def start(self):
+ """ Start XML-RPC server and register service."""
+
+ self.logs['main'].info("Starting HOD service: %s ..." % self.name)
+
+ if self._xrc: self._xrc.serve_forever()
+ if self._cfg.has_key('register') and self._cfg['register']:
+ self._register_service()
+
+ def stop(self):
+ """ Stop XML-RPC server, unregister service and set stop flag. """
+
+ self.logs['main'].info("Stopping service...")
+ if self._xrc: self._xrc.stop()
+ self.__stop = True
+
+ return True
+
+ def status(self):
+ """Returns true, should be overriden."""
+
+ return True
+
+ def wait(self):
+ """Wait until stop method is called."""
+
+ while not self.__stop:
+ time.sleep(.1)
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py Fri Jan 4 10:20:17 2008
@@ -0,0 +1,740 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+"""hodLogger provides a customized interface to Python's core logging package.
+"""
+
+import sys, os, re, logging, logging.handlers, inspect, pprint, types
+from tcp import get_address_tuple
+
+fileFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s \
+%(module)s:%(lineno)s - %(message)s'
+
+streamFormatString = '%(levelname)s - %(message)s'
+
+debugStreamFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s \
+%(module)s:%(lineno)s - %(message)s'
+
+syslogFormatString = '(%(process)d) %(levelname)s/%(levelno)s \
+%(module)s:%(lineno)s - %(message)s'
+
+smtpFormatString = '[%(asctime)s] %(levelname)s/%(levelno)s \
+%(module)s:%(lineno)s\n\n%(message)s'
+
+fileFormater = logging.Formatter(fileFormatString)
+streamFormater = logging.Formatter(streamFormatString)
+debugStreamFormater = logging.Formatter(debugStreamFormatString)
+syslogFormater = logging.Formatter(syslogFormatString)
+smtpFormater = logging.Formatter(smtpFormatString)
+
+defaultFileLevel = 3
+defaultStreamLevel = 4
+defaultSyslogLevel = 3
+defaultSmtpLevel = 0
+
+hodLogLevelMap = { 0 : logging.CRITICAL,
+ 1 : logging.ERROR,
+ 2 : logging.WARNING,
+ 3 : logging.INFO,
+ 4 : logging.DEBUG }
+
+hodStreamFormatMap = { 0 : streamFormater,
+ 1 : streamFormater,
+ 2 : streamFormater,
+ 3 : streamFormater,
+ 4 : debugStreamFormater }
+
+rehodLogLevelMap = {}
+for key in hodLogLevelMap.keys():
+ rehodLogLevelMap[hodLogLevelMap[key]] = key
+
+
+reModule = re.compile("^(.*)\..*$")
+
+hodLogs = {}
+
+class hodRotatingFileHandler(logging.handlers.RotatingFileHandler):
+ """ This class needs to be used in place of RotatingFileHandler when
+ the 2.4.0 Python interpreter is used."""
+
+ def emit(self, record):
+ """
+ Emit a record.
+
+ If a formatter is specified, it is used to format the record.
+ The record is then written to the stream with a trailing newline
+ [N.B. this may be removed depending on feedback]. If exception
+ information is present, it is formatted using
+ traceback.print_exception and appended to the stream.
+
+ *****
+
+ THIS IS A HACK, when instances of hodLogger get passed to the child of
+ a child thread for some reason self.stream gets closed. This version
+ of emit re-opens self.stream if it is closed. After testing it appears
+ that self.stream is only closed once after the second thread is
+ initialized so there is not performance penalty to this hack. This
+ problem only exists in python 2.4.
+
+ *****
+ """
+ try:
+ if self.shouldRollover(record):
+ self.doRollover()
+ try:
+ msg = self.format(record)
+ fs = "%s\n"
+ if not hasattr(types, "UnicodeType"): #if no unicode support...
+ self.stream.write(fs % msg)
+ else:
+ try:
+ self.stream.write(fs % msg)
+ except UnicodeError:
+ self.stream.write(fs % msg.encode("UTF-8"))
+ except ValueError:
+ self.stream = open(self.baseFilename, self.mode)
+ self.stream.write(fs % msg)
+
+ self.flush()
+ except:
+ self.handleError(record)
+ except:
+ self.handleError(record)
+
+ def shouldRollover(self, record):
+ """
+ Determine if rollover should occur.
+
+ Basically, see if the supplied record would cause the file to exceed
+ the size limit we have.
+
+ *****
+
+ THIS IS A HACK, when instances of hodLogger get passed to the child of
+ a child thread for some reason self.stream gets closed. This version
+ of emit re-opens self.stream if it is closed. After testing it appears
+ that self.stream is only closed once after the second thread is
+ initialized so there is not performance penalty to this hack. This
+ problem only exists in python 2.4.
+
+ *****
+ """
+ if self.maxBytes > 0: # are we rolling over?
+ msg = "%s\n" % self.format(record)
+
+ try:
+ #due to non-posix-compliant Windows feature
+ self.stream.seek(0, 2)
+ except ValueError:
+ self.stream = open(self.baseFilename, self.mode)
+ self.stream.seek(0, 2)
+
+ if self.stream.tell() + len(msg) >= self.maxBytes:
+ return 1
+ return 0
+
+class hodCustomLoggingLogger(logging.Logger):
+ """ Slight extension of the logging.Logger class used by the hodLog class.
+ """
+ def findCaller(self):
+ """ findCaller() is supposed to return the callers file name and line
+ number of the caller. This was broken when the logging package was
+ wrapped by hodLog. We should return much more relevant info now.
+ """
+
+ callerModule = ''
+ callerLine = 0
+
+ currentModule = os.path.basename(__file__)
+ currentModule = reModule.sub("\g<1>", currentModule)
+
+ frames = inspect.stack()
+ for i in range(len(frames)):
+ frameModule = os.path.basename(frames[i][1])
+ frameModule = reModule.sub("\g<1>", frameModule)
+ if frameModule == currentModule:
+ previousFrameModule = os.path.basename(frames[i+1][1])
+ previousFrameModule = reModule.sub("\g<1>",
+ previousFrameModule)
+ callerFile = frames[i+1][1]
+ callerLine = frames[i+1][2]
+ continue
+
+ returnValues = (callerFile, callerLine)
+ if sys.version.startswith('2.4.4') or sys.version.startswith('2.5'):
+ returnValues = (callerFile, callerLine, None)
+
+ return returnValues
+
+class hodLog:
+ """ Cluster management logging class.
+
+ logging levels: 0 - log only critical messages
+ 1 - log critical and error messages
+ 2 - log critical, error, and warning messages
+ 3 - log critical, error, warning, and info messages
+ 4 - log critical, error, warning, info, and debug
+ messages"""
+
+ def __init__(self, appName):
+ """Constructs a hodLogger object.
+
+ appName - name of logging application, log filenames will be
+ prepended with this name"""
+
+ self.__appName = appName
+
+ # initialize a dictionary to hold loggerNames
+ self.__loggerNames = {}
+
+ # initialize a dictionary to track log handlers and handler classes
+ self.__logObjs = { 'file' : {}, 'smtp' : {},
+ 'syslog' : {}, 'strm' : {} }
+
+ # use a custom logging.Logger class
+ logging.setLoggerClass(hodCustomLoggingLogger)
+
+ # get the root app logger
+ self.__logger = logging.getLogger(appName)
+ self.__logger.setLevel(logging.DEBUG)
+
+ hodLogs[self.__appName] = self
+
+ def __attr__(self, attrname):
+ """loggerNames - list of defined logger names"""
+
+ if attrname == "loggerNames": return self.__loggerNames.keys()
+ else: raise AttributeError, attrname
+
+ def __repr__(self):
+ """Returns a string representation of a hodLog object of the form:
+
+ LOG_NAME
+ file: FILENAME (level LEVEL)
+ smtp: SMTP_SERVER from FROM_ADDRESS (level LEVEL)
+ strm: STRM_OBJECT (level LEVEL)
+ ... """
+
+ hodLogString = "hodLog: %s\n\n" % self.__appName
+ for loggerName in self.__loggerNames.keys():
+ hodLogString = "%s logger: %s\n" % (hodLogString, loggerName)
+ handlerClasses = self.__logObjs.keys()
+ handlerClasses.sort()
+ for handlerClass in handlerClasses:
+ try:
+ loggerLevelName = logging.getLevelName(
+ self.__logObjs[handlerClass][loggerName]['level'])
+ hodLogString = "%s %s: %s (level %s)\n" % (
+ hodLogString, handlerClass,
+ self.__logObjs[handlerClass][loggerName]['data'],
+ loggerLevelName)
+ except:
+ hodLogString = "%s %s: none\n" % (
+ hodLogString, handlerClass)
+ hodLogString = "%s\n" % hodLogString
+
+ return hodLogString
+
+ # 'private' method which adds handlers to self.__logObjs
+ def __add_to_handlers(self, handlerClass, loggerName, handler, data,
+ level):
+ self.__logObjs[handlerClass][loggerName] = {}
+ self.__logObjs[handlerClass][loggerName]['handler'] = handler
+ self.__logObjs[handlerClass][loggerName]['data'] = data
+ self.__logObjs[handlerClass][loggerName]['level'] = level
+
+ # 'private' method which determines whether a hod log level is valid and
+ # returns a valid logging.Logger level
+ def __get_logging_level(self, level, defaultLevel):
+ loggingLevel = ''
+ try:
+ loggingLevel = hodLogLevelMap[int(level)]
+ except:
+ loggingLevel = hodLogLevelMap[defaultLevel]
+
+ return loggingLevel
+
+ # make a logging.logger name rootLogger.childLogger in our case the
+ # appName.componentName
+ def __get_logging_logger_name(self, loggerName):
+ return "%s.%s" % (self.__appName, loggerName)
+
+ def add_logger(self, loggerName):
+ """Adds a logger of name loggerName.
+
+ loggerName - name of component of a given application doing the
+ logging
+
+ Returns a hodLogger object for the just added logger."""
+
+ try:
+ self.__loggerNames[loggerName]
+ except:
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ logging.getLogger(loggingLoggerName)
+
+ self.__loggerNames[loggerName] = 1
+
+ return hodLogger(self.__appName, loggingLoggerName)
+
+ def add_file(self, logDirectory, maxBytes=0, backupCount=0,
+ level=defaultFileLevel, addToLoggerNames=None):
+ """Adds a file handler to all defined loggers or a specified set of
+ loggers. Each log file will be located in logDirectory and have a
+ name of the form appName-loggerName.log.
+
+ logDirectory - logging directory
+ maxBytes - maximum log size to write in bytes before rotate
+ backupCount - number of rotated logs to keep
+ level - cluster management log level
+ addToLoggerNames - list of logger names to which stream handling
+ will be added"""
+
+ def add_file_handler(loggerName):
+ if not self.__logObjs['file'].has_key(loggerName):
+ loggingLevel = self.__get_logging_level(level,
+ defaultFileLevel)
+
+ logFile = os.path.join(logDirectory, "%s-%s.log" % (
+ self.__appName, loggerName))
+
+ if sys.version.startswith('2.4'):
+ fileHandler = hodRotatingFileHandler(logFile,
+ maxBytes=maxBytes, backupCount=backupCount)
+ else:
+ fileHandler = logging.handlers.RotatingFileHandler(logFile,
+ maxBytes=maxBytes, backupCount=backupCount)
+
+ fileHandler.setLevel(loggingLevel)
+ fileHandler.setFormatter(fileFormater)
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ aLogger = logging.getLogger(loggingLoggerName)
+ aLogger.addHandler(fileHandler)
+
+ fileData = "%s" % logFile
+ self.__add_to_handlers('file', loggerName, fileHandler,
+ fileData, loggingLevel)
+
+ if addToLoggerNames:
+ for loggerName in addToLoggerNames:
+ add_file_handler(loggerName)
+ else:
+ for loggerName in self.__loggerNames:
+ add_file_handler(loggerName)
+
+ def add_stream(self, stream=sys.stderr, level=defaultStreamLevel,
+ addToLoggerNames=None):
+ """Adds a stream handler to all defined loggers or a specified set of
+ loggers.
+
+ stream - a stream such as sys.stderr or sys.stdout
+ level - cluster management log level
+ addToLoggerNames - tupple of logger names to which stream handling
+ will be added"""
+
+ def add_stream_handler(loggerName):
+ if not self.__logObjs['strm'].has_key(loggerName):
+ loggingLevel = self.__get_logging_level(level,
+ defaultStreamLevel)
+
+ streamHandler = logging.StreamHandler(stream)
+
+ streamHandler.setLevel(loggingLevel)
+
+ streamHandler.setFormatter(hodStreamFormatMap[int(level)])
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ aLogger = logging.getLogger(loggingLoggerName)
+ aLogger.addHandler(streamHandler)
+
+ streamData = "%s" % stream
+ self.__add_to_handlers('strm', loggerName, streamHandler,
+ streamData, loggingLevel)
+
+ if addToLoggerNames:
+ for loggerName in addToLoggerNames:
+ add_stream_handler(loggerName)
+ else:
+ for loggerName in self.__loggerNames:
+ add_stream_handler(loggerName)
+
+ def add_syslog(self, address, level=defaultSyslogLevel,
+ addToLoggerNames=None):
+ def add_syslog_handler(loggerName):
+ if not self.__logObjs['syslog'].has_key(loggerName):
+ loggingLevel = self.__get_logging_level(level,
+ defaultStreamLevel)
+
+ address[1] = int(address[1])
+ syslogHandler = logging.handlers.SysLogHandler(tuple(address),
+ 9)
+
+ syslogHandler.setLevel(loggingLevel)
+
+ syslogHandler.setFormatter(syslogFormater)
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ aLogger = logging.getLogger(loggingLoggerName)
+ aLogger.addHandler(syslogHandler)
+
+ syslogData = "%s:%s" % (address[0], address[1])
+ self.__add_to_handlers('syslog', loggerName, syslogHandler,
+ syslogData, loggingLevel)
+
+ if addToLoggerNames:
+ for loggerName in addToLoggerNames:
+ add_syslog_handler(loggerName)
+ else:
+ for loggerName in self.__loggerNames:
+ add_syslog_handler(loggerName)
+
+
+ def add_smtp(self, mailHost, fromAddress, toAddresses,
+ level=defaultSmtpLevel, addToLoggerNames=None):
+ """Adds an SMTP handler to all defined loggers or a specified set of
+ loggers.
+
+ mailHost - SMTP server to used when sending mail
+ fromAddress - email address to use as the from address when
+ sending mail
+ toAdresses - comma seperated list of email address to which
+ mail will be sent
+ level - cluster management log level
+ addToLoggerNames - tupple of logger names to which smtp handling
+ will be added"""
+
+ def add_email_handler(loggerName):
+ if not self.__logObjs['smtp'].has_key(loggerName):
+ loggingLevel = self.__get_logging_level(level,
+ defaultSmtpLevel)
+
+ subject = loggerName
+ if loggingLevel == 50:
+ subject = "%s - a critical error has occured." % subject
+ elif loggingLevel == 40:
+ subject = "%s - an error has occured." % subject
+ elif loggingLevel == 30:
+ subject = "%s - warning message." % subject
+ elif loggingLevel == 20:
+ subject = "%s - information message." % subject
+ elif loggingLevel == 10:
+ subject = "%s - debugging message." % subject
+
+ mailHostTuple = get_address_tuple(mailHost)
+ emailHandler = logging.handlers.SMTPHandler(mailHostTuple,
+ fromAddress, toAddresses, subject)
+
+ emailHandler.setFormatter(smtpFormater)
+ emailHandler.setLevel(loggingLevel)
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ aLogger = logging.getLogger(loggingLoggerName)
+ aLogger.addHandler(emailHandler)
+
+ emailData = "%s from %s" % (mailHost, fromAddress)
+ self.__add_to_handlers('smtp', loggerName, emailHandler,
+ emailData, loggingLevel)
+
+ if addToLoggerNames:
+ for loggerName in addToLoggerNames:
+ add_email_handler(loggerName)
+ else:
+ for loggerName in self.__loggerNames:
+ add_email_handler(loggerName)
+
+ def status(self):
+ statusStruct = {}
+ for loggerName in self.__loggerNames.keys():
+ statusStruct[loggerName] = []
+ for handlerClass in self.__logObjs.keys():
+ loggerDict = {}
+ try:
+ level = self.__logObjs[handlerClass][loggerName]['level']
+ level = rehodLogLevelMap[level]
+
+ loggerDict['handler'] = handlerClass
+ loggerDict['level'] = level
+ loggerDict['data'] = \
+ self.__logObjs[handlerClass][loggerName]['data']
+ except:
+ pass
+ else:
+ statusStruct[loggerName].append(loggerDict)
+
+ return statusStruct
+
+ def lock_handlers(self):
+ for handlerClass in self.__logObjs.keys():
+ for loggerName in self.__logObjs[handlerClass].keys():
+ self.__logObjs[handlerClass][loggerName]['handler'].acquire()
+
+ def release_handlers(self):
+ for handlerClass in self.__logObjs.keys():
+ for loggerName in self.__logObjs[handlerClass].keys():
+ self.__logObjs[handlerClass][loggerName]['handler'].release()
+
+ def get_level(self, handler, loggerName):
+ return rehodLogLevelMap[self.__logObjs[handler][loggerName]['level']]
+
+ def set_level(self, handler, loggerName, level):
+ """Sets the logging level of a particular logger and logger handler.
+
+ handler - handler (smtp, file, or stream)
+ loggerName - logger to set level on
+ level - level to set logger
+ """
+
+ level = self.__get_logging_level(level, defaultFileLevel)
+ self.__logObjs[handler][loggerName]['handler'].setLevel(level)
+ self.__logObjs[handler][loggerName]['level'] = level
+
+ if handler == 'stream':
+ self.__logObjs[handler][loggerName]['handler'].setFormatter(
+ hodStreamFormatMap[int(level)])
+
+ def set_logger_level(self, loggerName, level):
+ status = 0
+ for handlerClass in self.__logObjs.keys():
+ if self.__logObjs[handlerClass].has_key(loggerName):
+ self.set_level(handlerClass, loggerName, level)
+ else:
+ status = 1
+
+ return status
+
+ def rollover(self, loggerName):
+ status = 0
+ if self.__logObjs['file'].has_key(loggerName):
+ if self.__logObjs['file'][loggerName]['handler'].shouldRollover():
+ self.__logObjs['file'][loggerName]['handler'].doRollover()
+ else:
+ status = 1
+
+ return status
+
+ def set_max_bytes(self, maxBytes):
+ status = 0
+ if self.__logObjs.has_key('file'):
+ for loggerName in self.__logObjs['file'].keys():
+ self.__logObjs['file'][loggerName]['handler'].maxBytes = 0
+ else:
+ status = 1
+
+ return status
+
+ def get_logger(self, loggerName):
+ """ Returns a hodLogger object for a logger by name. """
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ return hodLogger(self.__appName, loggingLoggerName)
+
+ def critical(self, loggerName, msg):
+ """Logs a critical message and flushes log buffers. This method really
+ should only be called upon a catastrophic failure.
+
+ loggerName - logger to use
+ msg - message to be logged"""
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ logger = logging.getLogger(loggingLoggerName)
+ logger.critical(msg)
+ self.flush()
+
+ def error(self, loggerName, msg):
+ """Logs an error message and flushes log buffers.
+
+ loggerName - logger to use
+ msg - message to be logged"""
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ logger = logging.getLogger(loggingLoggerName)
+ logger.error(msg)
+ self.flush()
+
+ def warn(self, loggerName, msg):
+ """Logs a warning message.
+
+ loggerName - logger to use
+ msg - message to be logged"""
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ logger = logging.getLogger(loggingLoggerName)
+ logger.warn(msg)
+
+ def info(self, loggerName, msg):
+ """Logs an information message.
+
+ loggerName - logger to use
+ msg - message to be logged"""
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ logger = logging.getLogger(loggingLoggerName)
+ logger.info(msg)
+
+ def debug(self, loggerName, msg):
+ """Logs a debugging message.
+
+ loggerName - logger to use
+ msg - message to be logged"""
+
+ loggingLoggerName = self.__get_logging_logger_name(loggerName)
+ logger = logging.getLogger(loggingLoggerName)
+ logger.debug(msg)
+
+ def flush(self):
+ """Flush all log handlers."""
+
+ for handlerClass in self.__logObjs.keys():
+ for loggerName in self.__logObjs[handlerClass].keys():
+ self.__logObjs[handlerClass][loggerName]['handler'].flush()
+
+ def shutdown(self):
+ """Shutdown all logging, flushing all buffers."""
+
+ for handlerClass in self.__logObjs.keys():
+ for loggerName in self.__logObjs[handlerClass].keys():
+ self.__logObjs[handlerClass][loggerName]['handler'].flush()
+ # Causes famous 'ValueError: I/O operation on closed file'
+ # self.__logObjs[handlerClass][loggerName]['handler'].close()
+
+class hodLogger:
+ """ Encapsulates a particular logger from a hodLog object. """
+ def __init__(self, appName, loggingLoggerName):
+ """Constructs a hodLogger object (a particular logger in a hodLog
+ object).
+
+ loggingLoggerName - name of a logger in hodLog object"""
+
+ self.__appName = appName
+ self.__loggerName = loggingLoggerName
+ self.__logger = logging.getLogger(self.__loggerName)
+
+ def __repr__(self):
+ """Returns a string representation of a hodComponentLog object."""
+
+ return "%s hodLog" % self.__loggerName
+
+ def __call__(self):
+ pass
+
+ def set_logger_level(self, loggerName, level):
+
+ return hodLogs[self.__appName].set_logger_level(loggerName, level)
+
+ def set_max_bytes(self, maxBytes):
+
+ return hodLogs[self.__appName].set_max_bytes(maxBytes)
+
+ def rollover(self):
+ return hodLogs[self.__appName].rollover(self.__loggerName)
+
+ def get_level(self, handler, loggerName):
+
+ return hodLogs[self.__appName].get_level(handler, loggerName)
+
+ def critical(self, msg):
+ """Logs a critical message and calls sys.exit(1).
+
+ msg - message to be logged"""
+
+ self.__logger.critical(msg)
+
+ def error(self, msg):
+ """Logs an error message.
+
+ msg - message to be logged"""
+
+ self.__logger.error(msg)
+
+ def warn(self, msg):
+ """Logs a warning message.
+
+ msg - message to be logged"""
+
+ self.__logger.warn(msg)
+
+ def info(self, msg):
+ """Logs an information message.
+
+ msg - message to be logged"""
+
+ self.__logger.info(msg)
+
+ def debug(self, msg):
+ """Logs a debugging message.
+
+ msg - message to be logged"""
+
+ self.__logger.debug(msg)
+
+class hodDummyLogger:
+ """ Dummy hodLogger class. Other hod classes requiring a hodLogger default
+ to this hodLogger if no logger is passed."""
+
+ def __init__(self):
+ """pass"""
+
+ pass
+
+ def __repr__(self):
+ return "dummy hodLogger"
+
+ def __call__(self):
+ """pass"""
+
+ pass
+
+ def set_logger_level(self, loggerName, level):
+
+ return 0
+
+ def set_max_bytes(self, loggerName, maxBytes):
+
+ return 0
+
+ def get_level(self, handler, loggerName):
+
+ return 4
+
+ def rollover(self):
+
+ return 0
+
+ def critical(self, msg):
+ """pass"""
+
+ pass
+
+ def error(self, msg):
+ """pass"""
+
+ pass
+
+ def warn(self, msg):
+ """pass"""
+
+ pass
+
+ def info(self, msg):
+ """pass"""
+
+ pass
+
+ def debug(self, msg):
+ """pass"""
+
+ pass
+
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py Fri Jan 4 10:20:17 2008
@@ -0,0 +1,45 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import urllib, urlparse, re
+
+from HTMLParser import HTMLParser
+
+class miniHTMLParser( HTMLParser ):
+
+ viewedQueue = []
+ instQueue = []
+
+ def setBaseUrl(self, url):
+ self.baseUrl = url
+
+ def getNextLink( self ):
+ if self.instQueue == []:
+ return None
+ else:
+ return self.instQueue.pop(0)
+
+ def handle_starttag( self, tag, attrs ):
+ if tag == 'a':
+ newstr = urlparse.urljoin(self.baseUrl, str(attrs[0][1]))
+ if re.search('mailto', newstr) != None:
+ return
+
+ if (newstr in self.viewedQueue) == False:
+ self.instQueue.append( newstr )
+ self.viewedQueue.append( newstr )
+
+
+
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py Fri Jan 4 10:20:17 2008
@@ -0,0 +1,26 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements. See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership. The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+from hodlib.NodePools.torque import TorquePool
+
+class NodePoolUtil:
+ def getNodePool(nodePoolDesc, cfg, log):
+ """returns a concrete instance of NodePool as configured by 'cfg'"""
+ npd = nodePoolDesc
+ name = npd.getName()
+ if name == 'torque':
+ return TorquePool(npd, cfg, log)
+
+ getNodePool = staticmethod(getNodePool)
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL