You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2016/11/01 14:23:27 UTC
svn commit: r1767500 - in /uima/uima-ducc/trunk:
src/main/admin/cron/ducc_watcher.crontab.example
src/main/admin/ducc_watcher src/main/admin/tools/ducc_watcher
uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
Author: degenaro
Date: Tue Nov 1 14:23:27 2016
New Revision: 1767500
URL: http://svn.apache.org/viewvc?rev=1767500&view=rev
Log:
UIMA-5053 DUCC ducc_watcher optional admin script to determine status and send notifications
- revised to be stand alone, with no dependencies on relative location of script or on ducc.properties
Added:
uima/uima-ducc/trunk/src/main/admin/cron/ducc_watcher.crontab.example
uima/uima-ducc/trunk/src/main/admin/tools/ducc_watcher (with props)
Removed:
uima/uima-ducc/trunk/src/main/admin/ducc_watcher
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
Added: uima/uima-ducc/trunk/src/main/admin/cron/ducc_watcher.crontab.example
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/cron/ducc_watcher.crontab.example?rev=1767500&view=auto
==============================================================================
--- uima/uima-ducc/trunk/src/main/admin/cron/ducc_watcher.crontab.example (added)
+++ uima/uima-ducc/trunk/src/main/admin/cron/ducc_watcher.crontab.example Tue Nov 1 14:23:27 2016
@@ -0,0 +1,25 @@
+
+# Example crontab to backup DUCC database
+
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+# Once per minute
+
+*/1 * * * * /home/ducc/ducc_runtime/admin/tools --target http://uima-ducc-demo.apache.org:42 --email user1@host1 user2@host2
\ No newline at end of file
Added: uima/uima-ducc/trunk/src/main/admin/tools/ducc_watcher
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/tools/ducc_watcher?rev=1767500&view=auto
==============================================================================
--- uima/uima-ducc/trunk/src/main/admin/tools/ducc_watcher (added)
+++ uima/uima-ducc/trunk/src/main/admin/tools/ducc_watcher Tue Nov 1 14:23:27 2016
@@ -0,0 +1,412 @@
+#! /usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+# +
+# + ducc_watcher
+# +
+# + purpose: send e-mail when a DUCC daemon state changes to not up
+# +
+# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+import ast
+import datetime
+import getpass
+import json
+import logging
+import logging.handlers
+import os
+import smtplib
+import socket
+import sys
+import time
+import urllib
+
+from HTMLParser import HTMLParser
+
+from optparse import HelpFormatter
+from optparse import OptionGroup
+from optparse import OptionParser
+
+# ----------------------------------------------
+
+# Extend OptionParser class
+class ExtendedOptionParser(OptionParser):
+ # override epilog formatter so
+ # that newlines are not deleted!
+ def format_epilog(self, formatter):
+ return self.epilog
+
+# ----------------------------------------------
+
+# parser for the system.daemons WS page
+class DuccHtmlParser(HTMLParser):
+
+ tr_state = False
+ daemon_state = None
+ daemon_name = None
+ daemons = {}
+
+ def get_daemons(self):
+ return self.daemons
+
+ def handle_starttag(self, tag, attrs):
+ if(tag == 'tr' ):
+ self.tr_state = True
+
+ def handle_endtag(self, tag):
+ if(tag == 'tr'):
+ self.tr_state = False
+ self.daemon_state = None
+ self.daemon_name = None
+
+ def handle_data(self, data):
+ if(self.tr_state):
+ if(self.daemon_state == None):
+ self.daemon_state = data
+ elif(self.daemon_name == None):
+ self.daemon_name = data
+ self.daemons[self.daemon_name] = self.daemon_state
+ for key in self.daemons:
+ value = self.daemons[key]
+
+# ----------------------------------------------
+
+name = 'ducc_watcher'
+
+flag_info = True
+logger = None
+
+port = '42133'
+
+path = None
+log_file = None
+state_file = None
+
+mail_host = 'localhost'
+email_list = None
+
+# produce a time stamp
+def get_timestamp():
+ tod = time.time()
+ timestamp = datetime.datetime.fromtimestamp(tod).strftime('%Y-%m-%d %H:%M:%S')
+ return timestamp
+
+# get the host running this script
+def get_host():
+ host = socket.gethostname()
+ return host
+
+# get the user running this script
+def get_user():
+ user = getpass.getuser()
+ return user
+
+# make directories, if not already existing
+def mkdirs(path):
+ debug('mkdirs: path='+path)
+ if(os.path.exists(path)):
+ return
+ try:
+ os.makedirs(path)
+ except Exception,e:
+ exception(e)
+
+# info message to log
+def info(text):
+ global logger
+ type = 'I'
+ line = get_timestamp()+' '+get_user()+'@'+get_host()+' '+type+' '+text
+ logger.info(line)
+ return line
+
+# debug message to log
+def debug(text):
+ global logger
+ type = 'D'
+ line = get_timestamp()+' '+get_user()+'@'+get_host()+' '+type+' '+text
+ logger.debug(line)
+ return line
+
+# error message to log
+def error(text):
+ global logger
+ type = 'E'
+ line = get_timestamp()+' '+get_user()+'@'+get_host()+' '+type+' '+text
+ logger.error(line)
+ return line
+
+# warn message to log
+def warn(text):
+ global logger
+ type = 'W'
+ line = get_timestamp()+' '+get_user()+'@'+get_host()+' '+type+' '+text
+ logger.warn(line)
+ return line
+
+# exit
+def exit(code):
+ text = 'exit code='+str(code)
+ email(text)
+ error(text)
+ sys.exit(code)
+
+# exception
+def exception(e):
+ line = error(str(e))
+ return line
+
+# epilog for --help
+def get_epilog():
+ epilog = ''
+ return epilog
+
+# debug is normally not set
+def validate_debug(options):
+ global logger
+ if(options.flag_debug):
+ logger.setLevel(logging.DEBUG)
+ else:
+ logger.setLevel(logging.INFO)
+
+# use /tmp/<userid> as log+state directory
+# unless --path is specified
+def validate_path(options):
+ if(options.path == None):
+ options.path = '/tmp'+'/'+get_user()
+ mkdirs(options.path)
+
+# setup rotating log file handler with
+# 8 versions of 8M bytes with base name
+# ducc_watcher.log unless --log-file specified
+def validate_log_file(options):
+ global name
+ global logger
+ log_file = options.path
+ if(not log_file.endswith('/')):
+ log_file = log_file + '/'
+ if(options.log_file == None):
+ log_file = log_file + name+'.log'
+ else:
+ log_file = log_file + options.log_file
+ handler = logging.handlers.RotatingFileHandler(
+ log_file, maxBytes=8*1024*1024, backupCount=8)
+ logger.addHandler(handler)
+ debug('log_file: '+log_file)
+
+# use ducc_watcher.state unless --state-file specified
+def validate_state_file(options):
+ global name
+ global state_file
+ state_file = options.path
+ if(not state_file.endswith('/')):
+ state_file = state_file + '/'
+ if(options.state_file == None):
+ state_file = state_file + name+'.state'
+ debug('state_file: '+state_file)
+
+# must specify --target host:port of WS for fetching
+# of daemons status
+def validate_target(options):
+ global port
+ global target
+ global ducc_url
+ if(options.target == None):
+ error('required "target" not specified')
+ exit(1)
+ target = options.target
+ if(':' not in target):
+ target = target+':'+str(port)
+ if(target.startswith('http://')):
+ target = target.replace('http://','',1)
+ ducc_url = 'http://'+target+'/ducc-servlet/classic-system-daemons-data'
+ debug('target: '+ducc_url)
+
+# list of e-mail recipients, if any
+def validate_email_list(options):
+ global email_list
+ if(not options.email_list == None):
+ email_list = options.email_list.split()
+ debug('email-list: '+str(email_list))
+
+# parse command line
+def parse_cmdline():
+ global name
+ parser = ExtendedOptionParser(epilog=get_epilog())
+ width = 45
+ parser.formatter.help_position = width
+ parser.formatter.max_help_position = width
+ parser.add_option('-d','--debug', action='store_true', dest='flag_debug', default=False,
+ help='display debugging messages')
+ parser.add_option('-e','--email-list', action='store', dest='email_list', default=None,
+ help='blank separated list of email addresses to receive down + error notifications')
+ parser.add_option('-l','--log-file', action='store', dest='log_file', default=None,
+ help='name of watcher log file, default is '+name+'.log')
+ parser.add_option('-p','--path', action='store', dest='path', default=None,
+ help='path to directory where log and state information are written, default is /tmp'+'/'+get_user())
+ parser.add_option('-s','--state-file', action='store', dest='state_file', default=None,
+ help='name of watcher state file, default is '+name+'.state')
+ parser.add_option('-t','--target', action='store', dest='target', default=None,
+ help='[REQUIRED] <host> with default port of '+port+' or <host>:<port>')
+
+ (options, args) = parser.parse_args()
+ # -d
+ validate_debug(options)
+ # -e
+ validate_email_list(options)
+ # -p
+ validate_path(options)
+ # -l
+ validate_log_file(options)
+ # -s
+ validate_state_file(options)
+ # -t
+ validate_target(options)
+
+# read precious daemons state
+def read_state_previous():
+ global state_dict_previous
+ global state_file
+ state_dict_previous = {}
+ try:
+ with open(state_file, 'r') as f:
+ s = f.read()
+ state_dict_previous = ast.literal_eval(s)
+ debug('state_previous(read): '+str(state_dict_previous))
+ except Exception,e:
+ error('unable to read state from '+state_file)
+ exception(e)
+
+# write previous daemons state
+def write_state_previous():
+ global state_dict_previous
+ global state_file
+ try:
+ with open(state_file, 'w') as f:
+ f.seek(0)
+ f.write(str(state_dict_previous)+'\n')
+ f.truncate()
+ debug('state_previous(write): '+str(state_dict_previous))
+ except Exception,e:
+ error('unable to write state to '+state_file)
+ exception(e)
+
+# current becomes previous daemons state
+def update_state_previous():
+ global state_dict_previous
+ global state_dict_current
+ state_dict_previous = state_dict_current
+ write_state_previous()
+
+# fetch current daemons state
+def fetch_state_current():
+ global state_dict_current
+ global ducc_url
+ state_dict_current = {}
+ try:
+ response = urllib.urlopen(ducc_url)
+ data = response.read()
+ parser = DuccHtmlParser()
+ parser.feed(data)
+ daemons = parser.get_daemons()
+ if(daemons == None):
+ debug('daemons is None')
+ else:
+ for daemon in daemons:
+ status = daemons[daemon]
+ ##### <hack>
+ #status = 'down'
+ ##### </hack>
+ debug(daemon+':'+status+' ')
+ state_dict_current[daemon] = status
+ except Exception,e:
+ # for WS status to down whenever contact fails
+ daemon = 'Webserver'
+ status = 'unreachable'
+ state_dict_current[daemon] = status
+ error('unable to fetch data from '+ducc_url)
+ exception(e)
+ debug('state_current: '+str(state_dict_current))
+
+# determine state changes between previous and current
+def determine_state_changes():
+ global state_dict_current
+ global state_dict_previous
+ global state_dict_changes
+ state_dict_changes = {}
+ for key in state_dict_current:
+ state_current = state_dict_current.get(key, '?')
+ state_previous = state_dict_previous.get(key, '?')
+ if(state_current == state_previous):
+ pass
+ else:
+ info(key+' '+'from'+' '+state_previous+' '+'to'+' '+state_current)
+ if(state_current == 'up'):
+ pass
+ else:
+ state_dict_changes[key] = state_current
+ info(key+' '+state_current)
+
+# send email
+def email(text):
+ global name
+ global target
+ global ducc_url
+ global mail_host
+ global email_list
+ receivers = email_list
+ try:
+ if(text == None):
+ debug('no message?')
+ else:
+ sender = get_user()+'@'+get_host()
+ message = '['+sender+']'+' '+name+' '+'of'+' '+target+' '+'state change:'+' '+text
+ if(receivers == None):
+ info('nobody to send: '+text)
+ else:
+ smtpObj = smtplib.SMTP(mail_host)
+ smtpObj.sendmail(sender,receivers,message)
+ warn('sent: '+str(receivers)+' '+text)
+ except Exception,e:
+ error('not sent: '+str(receivers)+' '+text)
+ exception(e)
+ return
+
+# e-mail state changes, if any
+def email_state_changes():
+ global state_dict_changes
+ if(len(state_dict_changes) > 0):
+ email(str(state_dict_changes))
+
+# check for newly down DUCC daemons
+def main(argv):
+ global logger
+ logger = logging.getLogger('logger')
+ handler = logging.StreamHandler(sys.stdout)
+ logger.addHandler(handler)
+ parse_cmdline()
+ read_state_previous()
+ fetch_state_current()
+ determine_state_changes()
+ update_state_previous()
+ email_state_changes()
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
Propchange: uima/uima-ducc/trunk/src/main/admin/tools/ducc_watcher
------------------------------------------------------------------------------
svn:executable = *
Modified: uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex?rev=1767500&r1=1767499&r2=1767500&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex (original)
+++ uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex Tue Nov 1 14:23:27 2016
@@ -593,28 +593,20 @@ Red Hat Enterprise Linux Workstation rel
\begin{verbatim}
-$DUCC_HOME/admin/ducc_watcher is a Python script that, when run, contacts the
+$DUCC_HOME/admin/tools/ducc_watcher is a Python script that, when run, contacts the
DUCC Web Server to fetch data and determine the status of the critical head node daemons.
It can be run as a cron job to detect down daemons and send email notifications
-to a list of receipients specified in ducc.properties (via site.ducc.properties).
+to a list of receipients specified via command invocation option.
-Purpose: send e-mail notification when overall DUCC state changes
+Use the --help options for details.
-Configuration:
-
- - to ducc.properties (via site.ducc.properties) add:
- + ducc.watcher.notification.list = user1@host1 user2@host2
-
- - add $DUCC_HOME/admin/ducc_watcher as a cron job on a host that:
- + has access to $DUCC_HOME directory
- + has access to DUCC WebServer URL
- + has 'localhost' email server
+Purpose: send e-mail when a DUCC daemon state changes to not up
Files created by script:
- - $DUCC_HOME/state/watcher
- + comprises the last recorded overall state of DUCC
+ - /tmp/<user>/ducc_watcher.state
+ + comprises the last recorded state of DUCC daemons
- - $DUCC_HOME/logs/watcher.log
- + comprises a state change events log
+ - /tmp/<user>/ducc_watcher.log
+ + comprises a log produced by the script
\end{verbatim}
\ No newline at end of file