You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by cw...@apache.org on 2013/01/02 23:13:04 UTC
svn commit: r1428087 [1/4] - in /uima/sandbox/uima-ducc/trunk/src/main: ./
admin/ assembly/ config/ resources/ saxon/ scripts/
Author: cwiklik
Date: Wed Jan 2 22:13:03 2013
New Revision: 1428087
URL: http://svn.apache.org/viewvc?rev=1428087&view=rev
Log:
UIMA-2491
Added:
uima/sandbox/uima-ducc/trunk/src/main/
uima/sandbox/uima-ducc/trunk/src/main/admin/
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_diff
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_statedump
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_stomp.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/read_nodes.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_sim
uima/sandbox/uima-ducc/trunk/src/main/admin/verify_ducc
uima/sandbox/uima-ducc/trunk/src/main/assembly/
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml (with props)
uima/sandbox/uima-ducc/trunk/src/main/config/
uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml (with props)
uima/sandbox/uima-ducc/trunk/src/main/config/log4j.xml (with props)
uima/sandbox/uima-ducc/trunk/src/main/resources/
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.administrators
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties (with props)
uima/sandbox/uima-ducc/trunk/src/main/saxon/
uima/sandbox/uima-ducc/trunk/src/main/saxon/saxon8.jar (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/
uima/sandbox/uima-ducc/trunk/src/main/scripts/__init__.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/dd2spring.bat (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/dd2spring.xsl (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_boot.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_iface.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_perf_stats
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_qmon
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_service_cancel
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_service_submit
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_services
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_viewperf
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Wed Jan 2 22:13:03 2013
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+
+from ducc_boot import *
+set_ducc_home()
+
+
+import getopt
+import subprocess
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+
+class CheckDucc(DuccUtil):
+
+
+ def usage(self, msg):
+ if ( msg != None ):
+ print msg
+ print "Usage:"
+ print " check_ducc [options]"
+ print " If no options are given this is the equivalent of:"
+ print ""
+ print " check_ducc -n ../resources/ducc.nodes"
+ print ""
+ print "Options:"
+ print " -n --nodelist nodefile"
+ print " Check for agents on the nodes in nodefile. This option may be specified multiple time"
+ print " for multiple nodefiles. The 'local' node is always checked"
+ print ""
+ print " -u --user userid"
+ print " Userid is the user whose processes check_ducc searches for. If not specified,"
+ print " the user executing check_ducc is used. If specified as 'all' then all ducc processes"
+ print " are searched for."
+ print ""
+ print " -k --kill"
+ print " Force-kill any DUCC process you find on a node (if normal stop_ducc isn't working. This"
+ print " uses kill -9 and only kills processes owned by the invoking user."
+ print ""
+ print " -p --pids"
+ print " Rewrite the PID file. The PID file is always rewritten if any changes to processes are made. Sometimes"
+ print " the PID file needs rebuilding. This option causes the file to be rebuilt regardless of"
+ print " changes."
+ print ""
+ print " -r --reap"
+ print " Reap user processes. This uses kill -9 and ducc_ling to forcibly terrrminate user processes."
+ print " Only processes specified by '-u' or '--userid' are targeted."
+ print ""
+ print " -? prints this message."
+ sys.exit(1)
+
+ def main(self, argv):
+
+ self.show_ducc_environment()
+
+ try:
+ opts, args = getopt.getopt(argv, 'kn:pru:h?v')
+ except:
+ self.usage("Invalid arguments " + ' '.join(argv))
+
+ nodefiles = []
+ user = os.environ['LOGNAME']
+ kill = False
+ reap = False
+ redo_pids = False
+ process_changes = False
+
+ for ( o, a ) in opts:
+ if ( o == '-n' ) :
+ nodefiles.append(a)
+ elif ( o == '-k' ) :
+ kill = True
+ elif ( o == '-u' ) :
+ user = a
+ elif ( o == '-v'):
+ ducc_util.version()
+ elif ( o == '-r'):
+ reap = True
+ elif ( o == '-p'):
+ redo_pids = True
+ elif ( o == '-h'):
+ self.usage(None)
+ elif ( o == '-?'):
+ self.usage(None)
+ else:
+ print 'badarg', a
+ usage('bad arg: ' + a)
+
+
+ if ( reap and (user == 'ducc') ):
+ usage('Can only reap non-udcc users')
+
+
+ # init the PID file
+ pids = DuccProperties()
+ pids.load_if_exists(self.pid_file)
+
+ # read the nodelists
+ if ( len(nodefiles) == 0 ):
+ nodefiles = self.default_nodefiles
+ nodes = {}
+ for nf in nodefiles:
+ nodes = self.read_nodefile(nf, nodes)
+
+ #
+ # add in the local host if needed, and the webserver node
+ #
+ localnodes = []
+ if ( not self.localhost in nodes ):
+ localnodes.append(self.localhost)
+
+ if ( not (self.webserver_node in ['localhost', self.localhost, None]) ):
+ localnodes.append(self.webserver_node)
+
+ if ( len(localnodes) > 0 ):
+ nodes['local'] = localnodes
+
+ # checking starts here
+ for (nodefile, nodelist) in nodes.items():
+ if ( nodelist == None ):
+ # loading the nodes prints the necessary message
+ continue
+
+ for node in nodelist:
+
+ print 'Checking', node, '...',
+ proclist = self.find_ducc_process(node, user) # a list of tuples, tuple is (component, pid, user)
+ if ( len(proclist) > 0 ):
+ for proc in proclist:
+ component = proc[0]
+ pid = proc[1]
+ found_user = proc[2]
+
+ if ( component == 'orchestrator' ):
+ component = 'or'
+
+ if ( kill ) :
+ print 'Killing (kill -9)', component, '@', node, 'PID', pid
+ self.kill_process(node, proc)
+ pids.delete(pid)
+ process_changes = True
+ pass
+ elif (reap ):
+ # reaping is only for non-ducc JD and JP processes
+ if ( (component in ['jd', 'uima-as']) and (found_user != 'ducc') ):
+ print 'Reaping', component, found_user, pid
+ self.ssh(node, False, '/local/ducc/bin/ducc_ling', '-u', found_user, '--', '/bin/kill', '-9', pid)
+ else:
+ print '' # because the Checking message above is emitted with no \n
+ else:
+ print 'Found', component, '@', node, 'PID', pid, 'owned by', found_user
+ full_name = component + '@' + node
+ if ( component == 'agent' ):
+ pids.put(full_name, pid)
+
+ if ( component in self.default_components ):
+ pids.put(full_name, pid)
+ pids.put(component, full_name)
+ else:
+ print 'no processes found.'
+
+
+ if ( reap ):
+ return
+
+ if ( kill ):
+ self.remove_orchestrator_lock()
+
+ if ( len(pids) == 0):
+ if ( os.path.exists(self.pid_file) ):
+ os.remove(self.pid_file)
+ elif (process_changes or redo_pids):
+ pids.write(self.pid_file)
+
+if __name__ == "__main__":
+ checker = CheckDucc()
+ checker.main(sys.argv[1:])
+
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Wed Jan 2 22:13:03 2013
@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+import os
+import os.path
+import sys
+import getopt
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+
+class Ducc(DuccUtil):
+
+ def run_component(self, component, or_parms, numagents, rmoverride, background, nodup):
+
+ if ( component == 'all' ):
+ component = 'rm,sm,pm,ws,orchestrator'
+
+ complist = component.split(',')
+ args = None
+
+ jvm_opts = []
+ jvm_opts.append('-Dos.page.size=' + self.os_pagesize)
+ jvm_opts.append('-Dducc.deploy.configuration=' + self.DUCC_HOME + '/resources/ducc.properties')
+
+ service = 'org.apache.uima.ducc.common.main.DuccService'
+ for c in complist:
+ if ( c == 'agent' ):
+ if ( len(complist) > 1 ):
+ print "Must start agents separately"
+ sys.exit(1)
+
+ if ( not self.verify_duccling() ):
+ print 'ducc_ling is not set up correctly on node', self.localhost
+ return
+
+ jvm_opts.append('-Djava.library.path=' + self.DUCC_HOME)
+ if ( self.agent_jvm_args != None ):
+ jvm_opts.append(self.agent_jvm_args)
+
+ if ( (numagents > 1) ):
+ print '-------------------- launching special agent --------------------'
+ service = 'org.apache.uima.ducc.agent.launcher.Launcher'
+ args = ' ' + str(numagents)
+ jvm_opts.append('-DIP=192.168.3.85')
+ else:
+ ducc_component = '-Dducc.deploy.components=agent'
+
+ if ( c == 'rm' ):
+ if ( int(rmoverride) > 0 ):
+ jvm_opts.append("-Dducc.rm.override.dram=" + rmoverride)
+ if ( self.rm_jvm_args != None ):
+ jvm_opts.append(self.rm_jvm_args)
+
+ if ( c == 'ws' ):
+ here = os.getcwd()
+ os.chdir(self.DUCC_HOME + '/webserver')
+ if ( self.ws_jvm_args != None ):
+ jvm_opts.append(self.ws_jvm_args)
+
+ if ( c == 'viz' ):
+ here = os.getcwd()
+ os.chdir(self.DUCC_HOME + '/vizserver')
+ if ( self.ws_jvm_args != None ):
+ jvm_opts.append(self.ws_jvm_args)
+
+ if ( c == 'orchestrator' ):
+ if ( or_parms != '' ):
+ args = '-' + or_parms
+ if ( self.or_jvm_args != None ):
+ jvm_opts.append(self.or_jvm_args)
+
+ if ( c == 'pm' ):
+ if ( self.pm_jvm_args != None ):
+ jvm_opts.append(self.pm_jvm_args)
+
+ if ( c == 'sm' ):
+ if ( self.sm_jvm_args != None ):
+ jvm_opts.append(self.sm_jvm_args)
+
+
+ if (component != 'agent'):
+ service = 'org.apache.uima.ducc.common.main.DuccService'
+ ducc_component = '-Dducc.deploy.components=' + component
+
+ # check to see if there is a process like this running already, and barf if so
+ pid = None
+ if ( nodup ):
+ response = self.find_ducc_process(self.localhost, os.environ['LOGNAME']) # always make this "me"
+ if ( len(response) > 0 ):
+ for p in response:
+ if ( p[0] == component ):
+ print "Component", component,'is already running in PID', p[1], 'on node', self.localhost
+ pid = p[1]
+
+ cmd = []
+ cmd.append(self.java())
+ cmd.append(ducc_component)
+ cmd = cmd + jvm_opts
+ cmd.append(service)
+
+ if ( args != None ):
+ cmd.append(args)
+
+ #print 'CMD', cmd
+ if ( pid == None ):
+ if ( background ):
+ pid = self.nohup(cmd)
+ else:
+ pid = self.spawn(' '.join(cmd))
+ print 'PID ' + pid # nohup will print this from the (twice) forked process if background
+ # hard for us to access it here in nohup
+
+ if ( (c == 'ws') or ( c == 'viz') ):
+ os.chdir(here)
+
+ return
+
+ def usage(self, msg):
+ print msg
+ print 'Usage:'
+ print ' ducc.py -c <process> [-n <numagents>] [-b] [arguments ...]'
+ print ' ducc.py -k'
+ print 'Where:'
+ print ' -c <component> is the name of the comp[onent to start, currently one of'
+ print ' agent rm sm pm ws orchestrator'
+ print ' -- or --'
+ print ' all - to start rm sm pm ws orchestrator'
+ print ' NOTE -- that agents should be started separately'
+ print ' -b uses nohup and places the process into the background'
+ print ' -n <numagents> if > 1, multiple agents are started (testing mode)'
+ print ' -o <mem-in-GB> rm memory override for use on small machines'
+ print ' -k causes the entire DUCC system to shutdown'
+ print ' arguments - any additional arguments to pass to the component.'
+ sys.exit(1)
+
+ def main(self, argv):
+
+ component = None
+ numagents = 1
+ rmoverride = '0'
+ args = None
+ shutdown = False
+ background = False
+ or_parms = None
+ nodup = False # we allow duplicates unless asked not to
+
+ try:
+ opts, args = getopt.getopt(argv, 'bc:n:o:k?v', ['or_parms=', 'nodup'])
+ except:
+ self.usage('Bad arguments ' + ' '.join(argv))
+
+ for ( o, a ) in opts:
+ if ( o == '-c' ) :
+ component = a
+ if ( component == 'or' ):
+ component = 'orchestrator'
+
+ elif ( o == '-b'):
+ background = True
+ elif ( o == '-n'):
+ numagents = int(a)
+ elif ( o == '-o'):
+ rmoverride = a
+ elif ( o == '-k'):
+ shutdown = True
+ elif ( o == '--or_parms' ):
+ or_parms = a
+ elif ( o == '--nodup' ):
+ nodup = True
+ elif ( o == '-v'):
+ self.version()
+ else:
+ print 'badarg', a
+ usage('bad arg: ' + a)
+
+ if ( shutdown ):
+ if ( component != None ):
+ print 'Note: -c flag for component not allowed when shutting down. Shutdown aborted'
+ sys.exit(1);
+ self.clean_shutdown();
+ sys.exit(1)
+
+ if ( component == None ):
+ self.usage("Must specify component")
+
+ self.run_component(component, or_parms, numagents, rmoverride, background, nodup)
+ return
+
+ def __call__(self, *args):
+ self.main(args)
+ return
+
+if __name__ == "__main__":
+ ducc = Ducc()
+ ducc.main(sys.argv[1:])
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_diff
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_diff?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_diff (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_diff Wed Jan 2 22:13:03 2013
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+import getopt
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+from ducc import Ducc
+
+class PropsDiff(DuccUtil):
+
+ def usage(self, msg):
+ if (msg != None):
+ if ( msg[0] != None ):
+ msg = ' '.join(msg)
+ print msg
+
+ print "Usage:"
+ print " ducc_props_diff [other-props-file]"
+ print ''
+ print ' This script compares the installed ducc.properties against another ducc.properties'
+ sys.exit(1)
+
+ def main(self, argv):
+
+ if ( (len(argv) != 1) or (argv[0] == '-h') or (argv[0] == '-?') ):
+ self.usage(None)
+
+ diffs = DuccProperties()
+
+ foreign = DuccProperties();
+ try:
+ foreign.load(argv[0])
+ except:
+ print "Cannot load", argv[0]
+ sys.exit(1)
+
+ local = self.ducc_properties
+
+ # Iterate
+ # If a thing is in both maps, delete it from the maps
+ # and put it into the diffmap for printing
+ for ( k, v ) in foreign.items():
+ lv = local.get(k)
+ if ( lv != None ):
+ if ( v != lv ):
+ diffs.put(k, (v, lv))
+ local.delete(k)
+ foreign.delete(k)
+
+ print '--------------------------------------------------------------------------------'
+ if ( len(foreign) == 0 ):
+ print "Every property in", argv[0], "is in ducc.properties"
+ else:
+ print "These items are in", argv[0], "only"
+ for ( k, v ) in foreign.items():
+ print ' ', k, v
+ print '--------------------------------------------------------------------------------'
+ print ''
+
+ if ( local.items == 0 ):
+ print "Every property in ducc.properties is in", argv[0]
+ else:
+ print "These items are in ducc.properties only"
+ for ( k, v ) in local.items():
+ print ' ', k, v
+ print '--------------------------------------------------------------------------------'
+ print ''
+
+ print "These are in both maps with different values"
+ for ( k, v ) in diffs.items():
+ print k
+ print ' installed : ', v[1]
+ print ' compare to: ', v[0]
+ print ''
+
+if __name__ == "__main__":
+ diff = PropsDiff()
+ diff.main(sys.argv[1:])
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_statedump
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_statedump?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_statedump (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_statedump Wed Jan 2 22:13:03 2013
@@ -0,0 +1,137 @@
+#!/usr/bin/python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+from ducc_boot import *
+set_ducc_home()
+
+import getopt
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+from ducc import Ducc
+
+class DuccStateListener(DuccUtil):
+
+ def run(self):
+ CMD = self.java() + ' com.ibm.ducc.test.tools.DuccListener ' + self.host + ' ' + self.port + ' ' + self.topic + \
+ ' ' + self.agent + \
+ ' ' + self.output + \
+ ' ' + self.timeout
+ #print CMD
+ os.system(CMD)
+
+ def usage(self):
+ print "Usage:"
+ print " start_sim [options]"
+ print " If no options are given this help screen is shown."
+ print ""
+ print "Options:"
+ print " -n This is DUCC's ActiveMQ node, defaults to 'localhost'."
+ print ""
+ print " -p This is DUCC's ActiveMQ port, defaults to 61616."
+ print ""
+ print " -s This is the state to dump. One of rm sm or pm metrics inventory."
+ print ""
+ print " -t Timeout. or_statedump will run continuously for this long in seconds. Defaults to 300 seconds."
+ print ""
+ print " -o This is the name of a tempfile where the state is written, defaults to duccstate.out."
+ print ""
+ print ""
+ print "Remember that you will likely have to wait a few seconds for publications to arrive after starting."
+ print ""
+ print "To get a pretty-printed version of the xstream outpout."
+ print " xmllint --format duccstate.out --output duccstate.out.pretty"
+
+ sys.exit(0)
+
+ def main(self, argv):
+
+ self.host = 'localhost'
+ self.port = '61616'
+ self.output = 'none'
+ self.timeout = 'none'
+ self.state = 'or'
+ self.agent = "none"
+
+ try:
+ opts, args = getopt.getopt(argv, 'n:o:p:s:t:?h')
+ except:
+ self.invalid('Invalid arguments', ' '.join(argv))
+
+ for ( o, a ) in opts:
+ if o in ( '-n' ):
+ self.host = a
+ elif o in ( '-o' ):
+ self.output = a
+ elif o in ( '-p' ):
+ port = int(a) # quick check to see if it converts
+ self.port = a
+ elif o in ( '-s' ):
+ self.state = a
+ elif o in ( '-t' ):
+ timeout = int(a)
+ self.timeout = a
+ elif o in ( '-?', '-h' ):
+ self.usage()
+
+ CLASSPATH = os.environ['CLASSPATH']
+ CLASSPATH = CLASSPATH + ':' + self.DUCC_HOME + '/lib/ducc-test.jar'
+ os.environ['CLASSPATH'] = CLASSPATH
+
+ if ( self.state == 'or' ):
+ self.topic = 'ducc.orchestrator.state'
+ elif (self.state == 'rm' ):
+ self.topic = 'ducc.rm.state'
+ elif (self.state == 'sm' ):
+ self.topic = 'ducc.sm.state'
+ elif (self.state == 'pm' ):
+ self.topic = 'ducc.pm.state'
+ elif (self.state.startswith('inventory') ):
+ toks = self.state.split('@')
+ if ( len(toks) != 2 ):
+ print "Invalid state, must be 'inventory@node'"
+ sys.exit(1)
+ self.topic = 'ducc.node.inventory'
+ self.agent = toks[1]
+ elif (self.state.startswith('metrics') ):
+ toks = self.self.state.split('@')
+ print 'toks', toks
+ if ( len(toks) != 2 ):
+ print "Invalid state, must be 'metrics@node'"
+ sys.exit(1)
+ self.topic = 'ducc.node.metrics'
+ self.agent = toks[1]
+
+ print '-----------------------'
+ print 'host', self.host
+ print 'port', self.port
+ print 'timeout', self.timeout
+ print 'output file', self.output
+ print 'topic', self.topic
+ if ( self.agent != 'none' ):
+ print 'agent', self.agent
+ print '-----------------------'
+
+ self.run()
+
+if __name__ == "__main__":
+ dsl = DuccStateListener()
+ dsl.main(sys.argv[1:])
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_stomp.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_stomp.py?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_stomp.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_stomp.py Wed Jan 2 22:13:03 2013
@@ -0,0 +1,52 @@
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+import time
+import sys
+
+import logging
+import stomp
+
+from ducc_boot import *
+set_ducc_home()
+
+from ducc_util import DuccUtil
+
+class DuccStomp(DuccUtil):
+
+ def connect(self, host, port, listener):
+ self.host = host
+ self.port = port
+ self.listener = listener
+ logging.basicConfig()
+
+ conn = stomp.Connection( [(host, port)]) # list of host, port tuples
+ conn.set_listener('', listener)
+ conn.start()
+ conn.connect()
+ self.conn = conn
+
+ def close(self):
+ try:
+ self.conn.disconnect()
+ except:
+ sys.exit(0)
+
+ def subscribe(self, endpoint):
+ self.conn.subscribe(destination=endpoint, ack='auto')
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_stomp.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Jan 2 22:13:03 2013
@@ -0,0 +1,696 @@
+#!/usr/bin/python
+
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+import os
+import sys
+import string
+import subprocess
+import re
+import grp
+import zipfile
+import resource
+import time
+from stat import *
+
+class DuccPropertiesException(Exception):
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return repr(self.msg)
+
+class DuccProperties:
+ def __init__(self):
+ self.props = {}
+
+ #
+ # Expand ${} values from env or from this properties file itself
+ #
+ def do_subst(self, str):
+
+ key = None
+ p = re.compile("\\$\\{[a-zA-Z0-9_]+\\}")
+ m = p.match(str)
+
+ if ( m != None ):
+ key = m.group()[2:-1]
+ #print str, m, m.group(), key
+ val = os.environ[key]
+ response = string.replace(str, m.group() , val)
+ else:
+ response = str
+
+ return response
+
+ def mkitem(self, line):
+ ndx = line.find('#') # remove comments - like the java DuccProperties
+ if ( ndx >= 0 ):
+ line = line[0:ndx] # strip the comment
+ ndx = line.find('//') # remove comments - like the java DuccProperties
+ if ( ndx >= 0 ):
+ line = line[0:ndx] # strip the comment
+ line = line.strip() # clear leading and trailing whitespace
+ if ( line == '' ): # empty line?
+ return
+
+ mobj = re.search('[ =:]+', line)
+ if ( mobj ):
+ key = line[:mobj.start()].strip()
+ val = line[mobj.end():].strip()
+ #print 'NEXT', mobj.start(), 'END', mobj.end(), 'KEY', key, 'VAL', val
+ val = self.do_subst(val)
+ self.props[key] = val
+ else:
+ self.props[line] = None
+
+ #
+ # Load reads a properties file and adds it contents to the
+ # hash. It may be called several times; each call updates
+ # the internal has, thus building it up. The input file is
+ # in the form of a java-like properties file.
+ #
+ def load(self, propsfile):
+ if ( not os.path.exists(propsfile) ):
+ raise DuccPropertiesException(propsfile + ' does not exist and cannot be loaded.')
+
+ f = open(propsfile);
+ for line in f:
+ self.mkitem(line.strip())
+ f.close()
+
+ def load_from_manifest(self, jarfile):
+ z = zipfile.ZipFile(jarfile)
+ items = z.read('META-INF/MANIFEST.MF').split('\n')
+ for item in items:
+ self.mkitem(item)
+
+ #
+ # Try to load a properties file. Just be silent if it doesn't exist.
+ #
+ def load_if_exists(self, propsfile):
+ if ( os.path.exists(propsfile) ):
+ return self.load(propsfile)
+
+ #
+ # Put something into the hash.
+ #
+ def put(self, key, value):
+ self.props[key] = value
+
+ #
+ # Get something from the hash.
+ #
+ def get(self, key):
+ if ( self.props.has_key(key) ):
+ return self.props[key]
+ return None
+
+ #
+ # Remove an item if it exists
+ #
+ def delete(self, key):
+ if ( self.props.has_key(key) ):
+ del self.props[key]
+ #
+ # Write the has as a Java-like properties file
+ #
+ def write(self, propsfile):
+ f = open(propsfile, 'w')
+ items = self.props.items()
+ for (k, v) in items:
+ #print 'WRITING', k, '=', v
+ f.write(k + ' = ' + str(v) + '\n')
+ f.close()
+
+ #
+ # return a shallow copy of the dictionary
+ #
+ def copy_dictionary(self):
+ return self.props.copy()
+
+ #
+ # return the entries in the dictionary
+ #
+ def items(self):
+ return self.props.items()
+
+ #
+ # check to see if the key exists in the dictionary
+ #
+ def has_key(self, key):
+ return self.props.has_key(key)
+
+ #
+ # Return the length of the dictionary
+ #
+ def __len__(self):
+ return len(self.props)
+
+class DuccUtil:
+
+
+ def read_properties(self):
+
+ self.ducc_properties = DuccProperties()
+ self.ducc_properties.load(self.propsfile)
+
+ self.duccling = self.ducc_properties.get('ducc.agent.launcher.ducc_spawn_path')
+ self.webserver_node = self.ducc_properties.get('ducc.ws.node')
+ self.jvm = self.ducc_properties.get('ducc.jvm')
+ # self.broker_url = self.ducc_properties.get('ducc.broker.url')
+ self.broker_protocol = self.ducc_properties.get('ducc.broker.protocol')
+ self.broker_host = self.ducc_properties.get('ducc.broker.hostname')
+ self.broker_port = self.ducc_properties.get('ducc.broker.port')
+ self.broker_jmx_port = self.ducc_properties.get('ducc.broker.jmx.port')
+ self.broker_decoration = self.ducc_properties.get('ducc.broker.url.decoration')
+ self.broker_url = self.broker_protocol + '://' + self.broker_host + ':' + self.broker_port
+ self.agent_jvm_args = self.ducc_properties.get('ducc.agent.jvm.args')
+ self.ws_jvm_args = self.ducc_properties.get('ducc.ws.jvm.args')
+ self.pm_jvm_args = self.ducc_properties.get('ducc.pm.jvm.args')
+ self.rm_jvm_args = self.ducc_properties.get('ducc.rm.jvm.args')
+ self.sm_jvm_args = self.ducc_properties.get('ducc.sm.jvm.args')
+ self.or_jvm_args = self.ducc_properties.get('ducc.orchestrator.jvm.args')
+
+ if ( self.broker_decoration == '' ):
+ self.broker_decoration = None
+
+ if ( self.broker_decoration != None ):
+ self.broker_url = self.broker_url + '?' + self.broker_decoration
+
+ if ( self.webserver_node == None ):
+ self.webserver_node = self.localhost
+
+ def java(self):
+ if ( self.jvm == None ):
+ return 'java'
+ else:
+ return self.jvm
+
+ def is_amq_active(self):
+ lines = self.popen('ssh', self.broker_host, 'netstat -an')
+ #
+ # look for lines like this with the configured port in the 4th token, and
+ # ending with LISTEN:
+ #
+ # tcp 0 0 :::61616 :::* LISTEN
+ for line in lines:
+ toks = line.split()
+ if ( toks[-1] == 'LISTEN' ):
+ port = toks[3]
+ if (port.endswith(self.broker_port)):
+ return True
+ return False
+
+ def version(self):
+ lines = self.popen(self.jvm, ' org.apache.uima.ducc.utils.Version')
+ line = lines.readline().strip()
+ return "DUCC Version", line
+
+ def nohup_new(self, cmd, showpid=True):
+ nfds = resource.getrlimit(resource.RLIMIT_NOFILE)[1] # returns softlimit, hardlimit
+
+ # print 'NOHUP', cmd
+ print 'NOHUP', ' '.join(cmd)
+ print 'NOHUP', os.environ['IP']
+ print 'NOHUP', os.environ['NodeName']
+ #print 'NOHUP', os.environ['CLASSPATH']
+ try:
+ pid = os.fork()
+ except OSError, e:
+ raise Exception, "%s [%d]" % (e.strerror, e.errno)
+
+ if ( pid != 0 ):
+ return # the parent
+ else:
+ os.setsid()
+
+ try:
+ pid = os.fork()
+ except OSError, e:
+ raise Exception, "%s [%d]" % (e.strerror, e.errno)
+
+ if ( pid != 0 ):
+ if ( showpid ):
+ os.write(1, 'PID ' + str(pid) + '\n')
+ return
+
+ print 'NOHUP flushing'
+ sys.stdout.flush()
+ nfds = resource.getrlimit(resource.RLIMIT_NOFILE)[1] # returns softlimit, hardlimit
+ for i in range(3, nfds):
+ try:
+ #os.close(i);
+ pass
+ except:
+ pass # wasn't open
+
+ #devnull = os.devnull
+ #open(devnull, 'r') # fd 0 stdin
+ #open(devnull, 'w') # 1 stdout
+ #open(devnull, 'w') # 2 stderr
+ os.execvp(cmd[0], cmd)
+
+
+ def nohup(self, cmd, showpid=True):
+ cmd = ' '.join(cmd)
+ print '**** nohup', cmd, '****'
+ devnw = open(os.devnull, 'w')
+ devnr = open(os.devnull, 'r')
+ ducc = subprocess.Popen(cmd, shell=True, stdin=devnr, stdout=devnw, stderr=devnw)
+ devnr.close()
+ devnw.close()
+ if ( showpid ) :
+ print 'PID', ducc.pid
+
+ # simply spawn-and-forget using Python preferred mechanism
+ def spawn(self, *CMD):
+ cmd = ' '.join(CMD)
+ # print '**** spawn', cmd, '****'
+ ducc = subprocess.Popen(cmd, shell=True)
+ pid = ducc.pid
+ status = os.waitpid(pid, 0)
+ return pid
+
+ def popen(self, *CMD):
+ cmd = ' '.join(CMD)
+ #print 'POPEN:', cmd
+ proc = subprocess.Popen(cmd, bufsize=0, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT)
+ return proc.stdout
+
+ # like popen, only it spawns via ssh
+ def ssh(self, host, do_wait, *CMD):
+
+ cmd = ' '.join(CMD)
+ #print 'ssh -o BatchMode=yes -o ConnectTimeout=10', host, cmd
+ if ( do_wait ):
+ return self.popen('ssh -o BatchMode=yes -o ConnectTimeout=10', host, cmd)
+ else:
+ return self.spawn('ssh -o BatchMode=yes -o ConnectTimeout=10', host, cmd)
+
+
+ def set_classpath(self):
+ ducc_home = self.DUCC_HOME
+ LIB = ducc_home + '/lib'
+ RESOURCES = ducc_home + '/resources'
+
+ CLASSPATH = ''
+
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-activemq-5.5.0/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-commons-cli-1.2/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-commons-lang-2.6/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/guava-r09/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-log4j-1.2.16/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/uima/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-camel-2.7.1/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-commons-collections-3.2.1/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/joda-time-1.6/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/springframework-3.0.5/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/xmlbeans-2.5.0/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/bluepages/*'
+
+ # orchestrator http needs codecs
+ CLASSPATH = CLASSPATH + ":" + LIB + '/http-client/*'
+
+ # explicitly NOT ducc_test.jar
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-ibm.jar'
+ CLASSPATH = CLASSPATH + ':' + ducc_home + '/webserver/lib/*'
+ CLASSPATH = CLASSPATH + ':' + ducc_home + '/webserver/lib/jsp/*'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-agent.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-cli.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-common.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-jd.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-orchestrator.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-pm.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-rm.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-sm.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-web.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-viz.jar'
+
+ CLASSPATH = CLASSPATH + ':' + RESOURCES
+
+ os.environ['CLASSPATH'] = CLASSPATH
+
+ def set_classpath_for_clix(self):
+ ducc_home = self.DUCC_HOME
+ LIB = ducc_home + '/lib'
+ RESOURCES = ducc_home + '/resources'
+
+ CLASSPATH = ''
+
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-activemq-5.5.0/activemq-all-5.5.0.jar'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-commons-cli-1.2/commons-cli-1.2.jar'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/apache-camel-2.7.1/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/http-client/*'
+ CLASSPATH = CLASSPATH + ":" + LIB + '/springframework-3.0.5/*'
+
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-cli.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/ducc-common.jar'
+
+ CLASSPATH = CLASSPATH + ':' + LIB + '/uima/uima-core.jar'
+ CLASSPATH = CLASSPATH + ':' + LIB + '/uima/uimaj-as-core.jar'
+
+ CLASSPATH = CLASSPATH + ':' + RESOURCES
+
+ os.environ['CLASSPATH'] = CLASSPATH
+
+ def set_classpath_for_submit(self):
+ ducc_home = self.DUCC_HOME
+ LIB = ducc_home + '/lib'
+
+ CLASSPATH = LIB + '/ducc-submit.jar'
+ os.environ['CLASSPATH'] = CLASSPATH
+
+ def verify_duccling(self):
+
+ check_permission = True # if we're not ducc we don't care about permissions
+ user = os.environ['LOGNAME']
+ if ( user != 'ducc' ):
+ check_permission = False
+
+ if ( check_permission ) : # only care about ducc_ling setup if we're ducc
+ path = os.path.dirname(os.path.abspath(self.duccling))
+ dl = path + '/ducc_ling'
+
+ sstat = os.stat(path)
+ mode = sstat.st_mode
+ if ( not S_ISDIR(mode) ):
+ print 'ducc_ling path', path, ': Not a directory.'
+ return False
+
+ dirown = mode & (S_IRWXU | S_IRWXG | S_IRWXO)
+ #print 'Directory perms', oct(dirown)
+ if ( dirown != S_IRWXU ):
+ print 'ducc_ling path', path, ': Invalid directory permissions', oct(dirown), 'should be', oct(S_IRWXU)
+ return False
+
+ sstat = os.stat(dl)
+ mode = sstat.st_mode
+ expected = (S_IRWXU | S_IRGRP | S_IXGRP)
+ pathown = mode & (S_IRWXU | S_IRWXG | S_IRWXO)
+ #print 'Duccling perms', oct(pathown)
+ if ( pathown != expected ):
+ print 'ducc_ling module', dl, ': Invalid permissions', oct(pathown), 'Should be', oct(expected)
+ return False
+
+ if ( (mode & S_ISUID) != S_ISUID):
+ print 'ducc_ling module', dl, ': setuid bit is not set'
+ return False
+
+ try:
+ grpinfo = grp.getgrnam('ducc')
+ except:
+ print 'ducc_ling group "ducc" cannot be found.'
+ return false
+
+ duccgid = grpinfo.gr_gid
+ #print 'UID', sstat.st_uid, 'GID', duccgid
+ if ( (sstat.st_uid != 0) or (sstat.st_gid != duccgid) ):
+ print 'ducc_ling module', dl, ': Invalid ownership. Should be ducc.ducc'
+ return False
+
+ print 'ducc_ling OK'
+ return True
+
+ #
+ # Input is array lines from ps command looking for ducc processes owned this user.
+ # Output is list of dictionaries, where each dictionary describes a ducc process.
+ #
+ # If no ducc processes are found here the list is empty.
+ #
+ # The caller executes the 'ps' command and knows the node this is for.
+ #
+ def find_ducc_process(self, node, user):
+
+ answer = []
+ if ( user == 'all' ) :
+ resp = self.ssh(node, True, "'", 'ps auxw | grep java ', "'")
+ else:
+ resp = self.ssh(node, True, "'", 'ps auxw | grep ' + user + ' | grep java ', "'")
+
+ while True:
+ line = resp.readline().strip()
+ if ( line.startswith("Permission denied") ):
+ print node, "ALERT: Passwordless SSH is not configured correctly for node", node
+ print node, "ALERT: SSH returns '" + line + "'"
+ break
+
+ if ( line.startswith("Host key verification failed") ):
+ print node, "ALERT: Passwordless SSH is not configured correctly for node", node
+ print node, "ALERT: SSH returns '" + line + "'"
+ break
+
+ if ( line.find("Connection refused") >= 0 ):
+ print node, "ALERT: SSH is not not enabled on node", node
+ print node, "ALERT: SSH returns '" + line + "'"
+ break
+
+ if ( line.find("Connection timed") >= 0 ):
+ print node, "ALERT: SSH did not respond with timeout of 10 secnds", node
+ print node, "ALERT: SSH returns '" + line + "'"
+ break
+
+ if ( not line ):
+ break
+
+ toks = line.split()
+ if ( len(toks) < 11 ):
+ continue
+
+ pid = toks[1]
+ procname = toks[10]
+
+ if ( (user != 'all') and (toks[0] != user) ):
+ continue
+
+ if ( procname.endswith('java') ):
+ for tok in toks[10:]:
+ if ( tok.startswith('-Dducc.deploy.components=') ):
+ cmp = tok.split('=')
+ dp = (cmp[1], pid, toks[0])
+ answer.append(dp)
+
+ return answer
+
+ #
+ # Given the name of a file containing ducc nodes, a ducc user (usually 'ducc' unless you're running
+ # as yourself for test), find all ducc processes owned by this user and print them to the console.
+ #
+ def find_ducc(self, nodefile, user):
+ if ( nodefile == None ):
+ nodefile = self.DUCC_HOME + '/resources/ducc.nodes'
+
+ if ( not os.path.exists(nodefile) ):
+ print 'Nodefile', nodefile, 'does not exist or cannot be read.'
+ sys.exit(1)
+
+ answer = {}
+ nodes = []
+ f = open(nodefile)
+ for node in f:
+ node = node.strip()
+ if ( not node ):
+ continue
+ if ( node.startswith('#') ):
+ continue
+ nodes.append(node)
+
+ if ( self.webserver_node != 'localhost' ): # might be configured somewhere else
+ nodes.append(self.webserver_node)
+
+ for node in nodes:
+ data = self.find_ducc_process(node, user)
+ answer[node] = data
+
+ return answer
+
+
+
+ #def read_nodefile(self, nodefile, nodes):
+ #
+ # if ( not os.path.exists(nodefile) ):
+ # print 'Nodefile', nodefile, 'does not exist or cannot be read.'
+ # return None
+ #
+ # f = open(nodefile)
+ # for node in f:
+ # node = node.strip()
+ # if ( not node ):
+ # continue
+ # if ( node.startswith('#') ):
+ # continue
+ # nodes.append(node)
+ #
+ # return nodes
+
+ def remove_orchestrator_lock(self):
+ orlock = self.DUCC_HOME + '/state/orchestrator.lock'
+ try:
+ if ( os.path.exists(orlock) ):
+ os.remove(orlock)
+ print 'Orchestrator lock removed'
+ except:
+ print 'Unable to remove orchestrator lock'
+
+ def kill_process(self, node, proc):
+ self.ssh(node, False, 'kill', '-KILL', proc[1])
+
+ def clean_shutdown(self):
+ DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME + "/resources/ducc.properties "
+ self.spawn('java', DUCC_JVM_OPTS, 'org.apache.uima.ducc.common.main.DuccAdmin', '--killAll')
+
+ def get_os_pagesize(self):
+ lines = self.popen('/usr/bin/getconf', 'PAGESIZE')
+ return lines.readline().strip()
+
+ def show_ducc_environment(self):
+
+ #
+ # Print the java version
+ #
+ response = []
+ jvm = self.ducc_properties.get('ducc.jvm')
+ check_java = True
+ if ( jvm == None ):
+ response.append('WARNING: No jvm configured. Default is used.')
+ jvm = 'java'
+ else:
+ response.append('ENV: Java is configured as: ' + jvm)
+ if ( not os.path.exists(jvm) ):
+ print 'NOTOK: configured jvm cannot be found:', jvm
+ check_java = False
+
+ if ( check_java ):
+ lines = self.popen(jvm + ' -fullversion')
+ for line in lines:
+ response.append('ENV: ' + line.strip())
+
+
+ #
+ # Get the total memory for the node
+ #
+ meminfo = DuccProperties()
+ meminfo.load('/proc/meminfo')
+ mem = meminfo.get('MemTotal')
+ if ( mem.endswith('kB') ):
+ toks = mem.split(' ')
+ mem = str(int(toks[0]) / (1024*1024)) + ' gB'
+ response.append('MEM: memory is ' + mem)
+
+ #
+ # Get the operating system information
+ #
+ f = open('/proc/version')
+ for line in f:
+ response.append('ENV: system is ' + line.strip())
+ f.close()
+
+ #
+ # Print the version information from the DUCC jars
+ #
+ for j in [\
+ 'ducc-rm.jar',\
+ 'ducc-pm.jar', \
+ 'ducc-orchestrator.jar', \
+ 'ducc-sm.jar', \
+ 'ducc-web.jar', \
+ 'ducc-cli.jar', \
+ 'ducc-agent.jar', \
+ 'ducc-common.jar', \
+ 'ducc-jd.jar', \
+ 'ducc-test.jar', \
+ 'ducc-ibm.jar' \
+ ]:
+
+ manifest = DuccProperties()
+ manifest.load_from_manifest(self.DUCC_HOME + '/lib/' + j)
+ response.append('ENV: %25s %18s %12s %s' % (j + ':', manifest.get('Ducc-Version'), 'compiled at', manifest.get('Ducc-Build-Date')))
+
+ return response
+
+ #
+ # Resolve the 'path' relative to the path 'relative_to'
+ #
+ def resolve(self, path, relative_to):
+ if ( not path.startswith('/') ):
+ (head, tail) = os.path.split(os.path.abspath(relative_to))
+ path = head + '/' + path
+ return path
+
+ #
+ # Read the nodefile, recursing into 'imports' if needed, returning a
+ # map. The map is keyed on filename, with each entry a list of the nodes.
+ #
+ def read_nodefile(self, nodefile, ret):
+ #print 'READ_NODEFILE:', nodefile, ret
+ if ( os.path.exists(nodefile) ):
+ nodes = []
+ f = open(nodefile)
+ for node in f:
+ node = node.strip()
+ if ( not node ):
+ continue
+ if ( node.startswith('#') ):
+ continue
+ if ( node.startswith('import ') ):
+ toks = node.split(' ')
+ newfile = toks[1]
+ newfile = self.resolve(newfile, nodefile) # resolve newfile relative to nodefile
+ ret = self.read_nodefile(newfile, ret)
+ continue
+ nodes.append(node)
+ ret[nodefile] = nodes
+ else:
+ print 'Cannot read nodefile', nodefile
+ ret[nodefile] = None
+
+ #print 'RETURN', nodefile, ret
+ return ret
+
+ def __init__(self):
+
+ if ( os.environ.has_key('DUCC_HOME') ):
+ self.DUCC_HOME = os.environ['DUCC_HOME']
+ else:
+ me = os.path.abspath(sys.argv[0])
+ ndx = me.rindex('/')
+ ndx = me.rindex('/', 0, ndx)
+ self.DUCC_HOME = me[:ndx] # split from 0 to ndx
+ os.environ['DUCC_HOME'] = self.DUCC_HOME
+
+ self.jvm = None
+ self.webserver_node = 'localhost'
+ self.duccling = None
+ self.broker_url = 'tcp://localhost:61616'
+ self.broker_protocol = 'tcp'
+ self.broker_host = 'localhost'
+ self.broker_port = '61616'
+ self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'viz']
+ self.default_nodefiles = [self.DUCC_HOME + '/resources/ducc.nodes']
+ self.propsfile = self.DUCC_HOME + '/resources/ducc.properties'
+ self.localhost = os.uname()[1]
+ self.pid_file = self.DUCC_HOME + '/state/ducc.pids'
+ self.set_classpath()
+ self.read_properties()
+ self.os_pagesize = self.get_os_pagesize()
+
+if __name__ == "__main__":
+ util = DuccUtil()
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/read_nodes.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/read_nodes.py?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/read_nodes.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/read_nodes.py Wed Jan 2 22:13:03 2013
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+
+from ducc_boot import *
+set_ducc_home()
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+from ducc import Ducc
+
+#
+# Read the ducc node list and spew to stdout - handles comments, imports, etc.
+#
+class ReadNodes(DuccUtil):
+
+ def main(self, argv):
+ nodes = {}
+ nodes = self.read_nodefile(argv[0], nodes)
+ for ( nodefile, nodelist ) in nodes.items():
+ for host in nodelist:
+ print host
+
+if __name__ == "__main__":
+ if (len(sys.argv) == 1 ):
+ print "Usage: read_nodes <nodefile>"
+ sys.exit(1)
+
+ reader = ReadNodes()
+ reader.main(sys.argv[1:])
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/read_nodes.py
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Wed Jan 2 22:13:03 2013
@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+
+from ducc_boot import *
+set_ducc_home()
+
+import getopt
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+from ducc import Ducc
+
+class StartDucc(DuccUtil):
+
+ def start_component(self, ducc, component, or_parms):
+
+ print 'STARTING', or_parms
+ node = 'local'
+ com = component
+ if ( com.find('@') >= 0 ):
+ com, node = com.split('@')
+
+ if ( ((com == 'ws') or (com == 'viz')) and ( node == 'local' ) and ( self.webserver_node != 'localhost' )):
+ if ( self.webserver_node != None ):
+ node = self.webserver_node
+ component = com + '@' + node
+
+ if ((com in self.default_components) or ( com == 'agent')) :
+ print 'Starting', com, 'on', node,
+ else:
+ self.invalid('Unrecognized component', component)
+
+
+ if ( or_parms == None ):
+ or_parms = '--or_parms='
+ else:
+ or_parms = '--or_parms=' + or_parms
+
+ if ( node == 'local' ):
+ node = self.localhost
+
+ lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-b', or_parms, '--nodup' "'")
+ # we'll capture anything that the python shell spews because it may be useful, and then drop the
+ # pipe when we see a PID message
+ while 1:
+ line = lines.readline().strip()
+ if ( not line ):
+ break
+ # print '[] ' + line
+ if ( line.startswith('PID') ):
+ toks = line.split(' ') # get the PID
+ print 'PID', toks[1]
+ self.pids.put(com + '@' + node, toks[1])
+ lines.close()
+ break
+
+ if ( com in self.default_components ): # tracks where the management processes are
+ self.pids.put(com, com + '@' + node)
+
+ def start_agents(self, nodelist):
+
+ # print 'NODELIST', nodelist
+ #counter = 1
+ for host in nodelist:
+ #print 'COUNTER', counter
+ #counter = counter + 1
+ lines = self.ssh(host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c' 'agent', '-b', '--nodup', "'")
+ while 1:
+ line = lines.readline().strip()
+ # print '[]' + line
+ if ( line.startswith('PID') ):
+ toks = line.split(' ')
+ pid = toks[1]
+ self.pids.put('agent@' + host, pid)
+ lines.close()
+ print 'DUCC Agent started on node', host, 'PID', pid
+ break
+
+ if ( not line ):
+ break
+ toks = line.split()
+
+ # things that need checking:
+ # 1. Connection refused
+ # 2. "Add to keys" footprint message
+ # 3. Connection hang
+ # 4. - Passwordless not configured right
+ # 5. - no ducc.py on the other side
+ # 6. duccling not configured right on the other side
+ # 7. there is already a process of the given type on the other side
+ if ( toks[0] != 'ducc_ling' ):
+ if ( line.startswith("Permission denied") ):
+ print host, "ALERT: Passwordless SSH is not configured correctly for node"
+ print host, "ALERT: SSH returns '" + line + "'"
+ break
+
+ if ( line.find("No such file or directory") >= 0 ):
+ print host, "ALERT: ducc.py not found."
+ print host, "ALERT: SSH returns '" + line + "'"
+ break
+
+ print line
+ if ( line.find("Connection refused") >= 0 ):
+ print host, "ALERT: SSH is not not enabled on host", host
+ print host, "ALERT: SSH returns '" + line + "'"
+ break
+
+ if ( line.find("Connection timed") >= 0 ):
+ print host, "ALERT: SSH did not respond with timeout of 10 secnds", host
+ print host, "ALERT: SSH returns '" + line + "'"
+ break
+
+ #print '==========', line
+ continue
+
+ if ( toks[1] != 'OK' ):
+ print 'ALERT', host, line # 'line' has an appropriate error message
+
+
+ def usage(self, *msg):
+ if ( msg[0] != None ):
+ print ' '.join(msg)
+
+ print "Usage:"
+ print " start_ducc [options]"
+ print " If no options are given, all DUCC processes are started, using the default"
+ print " nodelist, DUCC_HOME/resources/ducc.nodes. This is the equivalemnt of"
+ print ""
+ print " start_ducc -n $DUCC_HOME/resources/ducc.nodes -m"
+ print ""
+ print "Options:"
+ print " -n --nodelist nodefile"
+ print " Start agents on the nodes in the nodefile. Multiple nodefiles may be specified:"
+ print ""
+ print " start_ducc -n foo.nodes -n bar.nodes -n baz.nodes"
+ print ""
+ print " -m --management"
+ print " Start the management processes (rm, sm, pm, webserver, orchestrator) on the local node."
+ print ""
+ print " -c, --component component"
+ print " Start a specific DUCC component, optionally on a specific node. If the component name"
+ print " is qualified with a nodename, the component is started on that node. To qualify a"
+ print " component name with a destination node, use the notation component@nodename."
+ print " Multiple components may be specified:"
+ print ""
+ print " start_ducc -c sm -c pm -c rm@f9n4 -c or@bluej22 -c agent@remote1 -c agent@remote2"
+ print ""
+ print " Components include:"
+ print " rm - resource manager"
+ print " or - orchestrator"
+ print " pm - process manager"
+ print " sm - services manager"
+ print " ws - web server"
+ print " viz - visualization server"
+ print " agent - node agent"
+ print ""
+ print "Examples:"
+ print " Start all DUCC processes, using custom nodelists:"
+ print " start_ducc -m -n foo.nodes -n bar.nodes"
+ print ""
+ print " Start just managemnet processes:"
+ print " start_ducc -m"
+ print ""
+ print " Start just agents on a specific set of nodes:"
+ print " start_ducc -n foo.nodes -n bar.nodes"
+ print ""
+ print " Start the webserver on node 'bingle':"
+ print " start_ducc -c ws@bingle"
+ sys.exit(1)
+
+ def invalid(self, *msg):
+ if ( msg[0] != None ):
+ print ' '.join(msg)
+
+ print "For usage run"
+ print " start_ducc -h"
+ print 'or'
+ print ' start_ducc --help'
+ sys.exit(1)
+
+ def main(self, argv):
+
+ environ = self.show_ducc_environment()
+ for e in environ:
+ print e
+
+ nodefiles = []
+ components = []
+ management = False
+ or_parms = self.ducc_properties.get('ducc.orchestrator.start.type')
+ self.pids = DuccProperties()
+ self.pids.load_if_exists(self.pid_file)
+
+ try:
+ opts, args = getopt.getopt(argv, 'c:mn:h?v', ['component=', 'components=', 'help', 'nodelist=', 'management', 'cold', 'warm', 'hot'])
+ except:
+ self.invalid('Invalid arguments', ' '.join(argv))
+
+ for ( o, a ) in opts:
+ if o in ( '-c', '--components' ):
+ components.append(a)
+ elif o in ( '-m', '--management' ):
+ management = True
+ elif o in ( '-n', '--nodelist' ):
+ nodefiles.append(a)
+ elif o in ( '--cold', '--warm', '--hot' ):
+ or_parms = o[2:] # (strip the leading --)
+ elif ( o == '-v'):
+ print self.version()
+ sys.exit(0)
+ elif o in ( '-h', '--help' ):
+ self.usage(None)
+ elif ( o == '-?'):
+ self.usage(None)
+ else:
+ self.invalid('bad args: ', ' '.join(argv))
+
+
+ # 'management' means start all the management daemons - if specific components are also specified
+ # there is at least a redundancy and maybe also a conflict.
+ if ( (len(components) != 0) and management ):
+ self.invalid("The --management and --compoent options are mutually exclusive")
+
+ # no args - make equivalent of -management and -nodefile=DUCC.HOME/resources/ducc.nodes
+ if ( len(argv) == 0 ):
+ nodefiles = self.default_nodefiles
+ components = self.default_components
+
+
+ # this means all the non-agent processes - conflicts are already checked
+ if ( management ):
+ components = self.default_components
+
+ #print 'nodefiles:', nodefiles
+ #print 'components:', components
+
+ # make sure all the nodefiles exist and are readable
+ ok = True
+ nodes = {}
+ for n in nodefiles:
+ nodes = self.read_nodefile(n, nodes)
+
+ for ( nf, nl ) in nodes.items():
+ if ( nl == None ):
+ print "Can't read nodefile", nf
+ ok = False
+
+ if ( not ok ):
+ sys.exit(1)
+
+ # activeMQ needs to be started externally before starting any DUCC processes
+ #print 'A--------------------------------------------------------------------------------'
+ if ( self.is_amq_active() ):
+ print 'ActiveMQ is found on configured host and port:', self.broker_host + ':' + self.broker_port
+ else:
+ print 'ActiveMQ cannot be found on configured host and port:', self.broker_host + ':' + self.broker_port
+ sys.exit(1)
+ #print 'B--------------------------------------------------------------------------------'
+
+ ducc = Ducc()
+
+ # if we are asked to start any of the managemnt processes, do this first
+ if ( len(components) != 0 ):
+ for com in components:
+ self.start_component(ducc, com, or_parms)
+ else:
+ print 'Not starting management components.'
+ #print 'C--------------------------------------------------------------------------------'
+
+ #print 'D--------------------------------------------------------------------------------'
+ for (nodefile, nodelist) in nodes.items():
+ print '********** Starting agents from file', nodefile
+ self.start_agents(nodelist)
+ #print 'E--------------------------------------------------------------------------------'
+
+ if ( len(self.pids) > 0 ):
+ self.pids.write(self.pid_file)
+ return
+
+if __name__ == "__main__":
+ starter = StartDucc()
+ starter.main(sys.argv[1:])
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim?rev=1428087&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim Wed Jan 2 22:13:03 2013
@@ -0,0 +1,291 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+import getopt
+
+from ducc_boot import *
+set_ducc_home()
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+from ducc import Ducc
+
+class StartSim(DuccUtil):
+
+ def run_local_agent(self, pnode, ip, memory ):
+
+ memory = int(memory) * 1024 * 1024 # to GB from KB
+ CMDPARMS = []
+ CMDPARMS.append(self.java())
+ CMDPARMS.append('-Dducc.deploy.components=agent')
+ CMDPARMS.append('-Dos.page.size=' + self.os_pagesize)
+ CMDPARMS.append('-Dducc.deploy.configuration=' + self.DUCC_HOME + "/resources/ducc.properties")
+ CMDPARMS.append('-Djava.library.path=' + self.DUCC_HOME)
+ CMDPARMS.append('-Xmx100M')
+ CMDPARMS.append('-Dducc.agent.node.metrics.fake.memory.size=' + str(memory))
+ CMDPARMS.append('org.apache.uima.ducc.common.main.DuccService')
+
+ print "Start agent with pnode", pnode, "IP", ip, "memory", memory
+ os.environ['NodeName'] = pnode
+ os.environ['IP'] = ip
+
+ self.nohup(CMDPARMS)
+ #self.spawn(' '.join(CMDPARMS))
+
+ #
+ # Start admin components rm pm sm ws or, on local node using Ducc.py
+ #
+ def startComponents(self, components, or_parms):
+ for (com, com) in components.items():
+ if ( com in ('ws', 'viz') ):
+ node = self.webserver_node
+ else:
+ node = self.localhost
+
+ if ( com == 'or' ):
+ lines = self.ssh(node, True, "'",
+ self.DUCC_HOME + '/admin/ducc.py', '-c', 'or', '-b',
+ '--or_parms', or_parms, "'")
+ else:
+ lines = self.ssh(node, True, "'",
+ self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-b', "'")
+
+ print 'Start', com, 'on', node,
+ while 1:
+ line = lines.readline().strip()
+ if ( not line ):
+ break
+ # print '[] ' + line
+ if ( line.startswith('PID') ):
+ toks = line.split(' ') # get the PID
+ self.pids.put(com, self.localhost + ' ' + toks[1] + ' ' + self.localhost)
+ lines.close()
+ print 'PID', toks[1]
+ break
+
+
+ #
+ # Read the special nodelist and start "special" agents
+ #
+ # Nodelist has records compatible with java properties like this:
+ #
+ # index nodename memory-in-gb
+ #
+ # We generate fake IP addresses from 192.168.4.X where x is the index.
+ # We generate fake node names from nodename-X where x is the index.
+ # We pass the memory-in-gb to the agent as the fake memory it reports instead of real memory.
+ #
+ # We use Jerry's memory override for each agent to get it to falsely report the memory
+ # instead of reading from the real machine.
+ #
+
+ def startAgents(self, nodelist, instances):
+
+
+ do_all = True
+ if ( len(instances) > 0 ):
+ do_all = False
+
+ for nf in nodelist:
+ print "Starting from nodes in", nf
+ props = DuccProperties()
+ props.load(nf)
+
+ for (index, details) in props.items():
+
+ if ( not do_all ):
+ if ( not instances.has_key(index) ):
+ continue
+
+ toks = details.split(' ')
+ ip = '192.168.4.' + index
+ node = toks[0]
+ mem = toks[1]
+ pnode = node + '-' + index
+ print 'Starting agent on', node, 'instance', index, 'as pseudo-node', pnode, 'IP', ip, 'memory', mem,
+
+ here = os.getcwd()
+ me = os.path.abspath(sys.argv[0])
+ cmd = 'export DUCC_HOME=' + self.DUCC_HOME + ';' + me
+ lines = self.ssh(node, True, "'", cmd, '--agent', '--memory', mem, '--addr', ip, '--pseudoname', pnode, "'")
+ while 1:
+ line = lines.readline().strip()
+ if ( not line ):
+ break
+ #print '[1]', line
+ if ( line.startswith('PID')):
+ toks = line.split(' ') # get the PID
+ lines.close()
+ print 'Started, PID', toks[1]
+ self.pids.put(index, node + ' ' + toks[1] + ' ' + pnode)
+ break
+
+
+ def usage(self, msg):
+ if (msg != None):
+ if ( msg[0] != None ):
+ msg = ' '.join(msg)
+ print msg
+
+ print "Usage:"
+ print " start_sim [options]"
+ print " If no options are given this help screen is shown."
+ print ""
+ print "Options:"
+ print " -n --nodelist nodelist"
+ print " The nodelist describing agents that is to be used. Not valid with --agent, --memory, --agent, or --pseudoname."
+ print " A nodelist provides the parameters for starting agents. Lines are of the form:"
+ print " index nodename mem-in-GB"
+ print " 'index' is any unique number."
+ print " 'nodename' is the physical node you want the agent placed on"
+ print " 'mem-in-GB' is the amount of simulated memory the agent will report"
+ print ""
+ print " Example:"
+ print " 1 bluej290 31"
+ print " 2 bluej290 31"
+ print " 9 bluej291 31"
+ print " 10 bluej291 31"
+ print " 17 bluej292 47"
+ print " 18 bluej292 47"
+ print " 25 bluej293 47"
+ print " 26 bluej293 47"
+ print ""
+ print " -c --components component"
+ print " Start the indicated component, must be one of", ' '.join(self.default_components), "or 'all'"
+ print ""
+ print " -i --instance instanceid"
+ print " Start only this instance of an agent from the nodelist."
+ print ""
+ print " --agent"
+ print " Start an agent only on localhost. All of --memory, --addr, and --pseudoname are required, -n is disallowed."
+ print ""
+ print " --memory mem"
+ print " Use this memory override. Valid only with --agent."
+ print ""
+ print " --addr"
+ print " Use this IP override. Valid only with --agent."
+ print ""
+ print " --pseudoname pseudoname"
+ print " Use this as the hostname for the agent. Valid only with -a."
+ print ""
+ print " -v, --version"
+ print " Print the current DUCC version"
+ sys.exit(1)
+
+
+ def invalid(self, *msg):
+ if ( msg != None ):
+ if ( msg[0] != None ):
+ msg = ' '.join(msg)
+ print msg
+
+ print "For usage run"
+ print " start_sim -h"
+ print 'or'
+ print ' start_sim --help'
+ sys.exit(1)
+
+ def main(self, argv):
+
+ if ( len(argv) == 0 ):
+ self.usage(None)
+
+ nodefiles = []
+ components = {}
+ instances = {}
+ run_agent = False
+ memory = None
+ pseudoname = None
+ IP = None
+
+ or_parms = self.ducc_properties.get('ducc.orchestrator.start.type')
+
+ try:
+ opts, args = getopt.getopt(argv, 'c:i:n:vh?', ['component=', 'help', 'agent', 'memory=',
+ 'instance=', 'addr=', 'pseudoname=', 'nodelist=',
+ 'version', 'hot', 'warm', 'cold'])
+ except:
+ self.invalid('Invalid arguments', ' '.join(argv))
+
+ for ( o, a ) in opts:
+ if o in ( '-n', '--nodelist' ):
+ nodefiles.append(a)
+ elif o in ( '--agent' ):
+ run_agent = True
+ elif o in ( '-c', '--component' ):
+ if ( a == 'all' ):
+ for cmp in self.default_components:
+ components[cmp] = cmp
+ else:
+ components[a] = a
+ elif o in ( '--memory' ):
+ memory = a
+ elif o in ( '--addr' ):
+ IP = a
+ elif o in ( '-i', '--instance' ):
+ instances[a] = a
+ elif o in ( '--pseudoname' ):
+ pseudoname = a
+ elif o in ( '-v', '--version' ):
+ print self.version()
+ os.exit(0)
+ elif o in ( '--hot', '--warm', '--cold' ):
+ or_parms = o[2:]
+ elif o in ( '-h', '--help' ):
+ self.usage(None)
+ elif ( o == '-?'):
+ self.usage(None)
+ else:
+ self.invalid('bad args: ', ' '.join(argv))
+
+ self.pids = DuccProperties()
+ self.ducc = Ducc()
+
+ if ( os.path.exists('sim.pids') ):
+ self.pids.load('sim.pids')
+ if ( run_agent ):
+ #
+ # checks that aren't valid if we want run_agent
+ #
+ if ( len(nodefiles) != 0 ):
+ self.invalid("Nodelist is not compatible with agent")
+
+ if ( (IP == None ) or ( memory == None) or ( pseudoname == None )):
+ self.invalid("Missing IP, memory, or pseudoname")
+
+ self. run_local_agent(pseudoname, IP, memory)
+ sys.exit(0)
+ else:
+ if ( (IP != None) or (memory != None) or ( pseudoname != None )) :
+ self.invalid("Running with a nodelist is not compatible with running a single agent.");
+
+ if ( len(nodefiles) != 0 ):
+ self.startAgents(nodefiles, instances)
+ self.startComponents(components, or_parms)
+
+ self.pids.write('sim.pids')
+
+if __name__ == "__main__":
+ starter = StartSim()
+ starter.main(sys.argv[1:])
+