You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2013/07/08 21:29:04 UTC
svn commit: r1500896 - /uima/sandbox/uima-ducc/trunk/src/main/scripts/
Author: challngr
Date: Mon Jul 8 19:29:04 2013
New Revision: 1500896
URL: http://svn.apache.org/r1500896
Log:
UIMA-3055 Update general-user scripting updates for refactor of ducc_util.py.
Added:
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_base.py (with props)
Removed:
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_boot.py
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_service_cancel
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_service_submit
Modified:
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_perf_stats
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_cancel
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_submit
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_services
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve
uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_web
Added: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_base.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_base.py?rev=1500896&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_base.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_base.py Mon Jul 8 19:29:04 2013
@@ -0,0 +1,275 @@
+#!/usr/bin/python
+
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+import os
+import sys
+import string
+import subprocess
+import re
+import zipfile
+import platform
+
+class DuccPropertiesException(Exception):
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return repr(self.msg)
+
+class DuccProperties:
+ def __init__(self):
+ self.props = {}
+
+ #
+ # Expand all ${} values from env or from this properties file itself
+ # The search order is:
+ # 1 look in this properties file
+ # 2 look in the environment
+ #
+ def do_subst(self, str):
+ key = None
+ p = re.compile("\\$\\{[a-zA-Z0-9_\\.\\-]+\\}")
+ ndx = 0
+
+ response = str
+ m = p.search(response, ndx)
+ while ( m != None ):
+ key = m.group()[2:-1]
+
+ val = None
+ if ( self.has_key(key) ):
+ val = self.get(key)
+ elif ( os.environ.has_key(key) ):
+ val = os.environ[key]
+
+ if ( val != None ):
+ response = string.replace(response, m.group() , val)
+ ndx = m.start()+1
+ m = p.search(response, ndx)
+
+ return response
+
+ def mkitem(self, line):
+ ndx = line.find('#') # remove comments - like the java DuccProperties
+ if ( ndx >= 0 ):
+ line = line[0:ndx] # strip the comment
+ ndx = line.find('//') # remove comments - like the java DuccProperties
+ if ( ndx >= 0 ):
+ line = line[0:ndx] # strip the comment
+ line = line.strip() # clear leading and trailing whitespace
+ if ( line == '' ): # empty line?
+ return
+
+ mobj = re.search('[ =:]+', line)
+ if ( mobj ):
+ key = line[:mobj.start()].strip()
+ val = line[mobj.end():].strip()
+ #print 'NEXT', mobj.start(), 'END', mobj.end(), 'KEY', key, 'VAL', val
+ # val = self.do_subst(val) # we'll do lazy subst on get instead
+ self.props[key] = val
+ else:
+ self.props[line] = None
+
+ #
+ # Load reads a properties file and adds it contents to the
+ # hash. It may be called several times; each call updates
+ # the internal has, thus building it up. The input file is
+ # in the form of a java-like properties file.
+ #
+ def load(self, propsfile):
+ if ( not os.path.exists(propsfile) ):
+ raise DuccPropertiesException(propsfile + ' does not exist and cannot be loaded.')
+
+ f = open(propsfile);
+ for line in f:
+ self.mkitem(line.strip())
+ f.close()
+
+ def load_from_manifest(self, jarfile):
+ z = zipfile.ZipFile(jarfile)
+ items = z.read('META-INF/MANIFEST.MF').split('\n')
+ for item in items:
+ self.mkitem(item)
+
+ #
+ # Try to load a properties file. Just be silent if it doesn't exist.
+ #
+ def load_if_exists(self, propsfile):
+ if ( os.path.exists(propsfile) ):
+ return self.load(propsfile)
+
+ #
+ # Put something into the hash.
+ #
+ def put(self, key, value):
+ self.props[key] = value
+
+ #
+ # Get something from the hash.
+ #
+ def get(self, key):
+ if ( self.props.has_key(key) ):
+ return self.do_subst(self.props[key]) # we'll do lazy subst on get instead
+ return None
+
+ #
+ # Remove an item if it exists
+ #
+ def delete(self, key):
+ if ( self.props.has_key(key) ):
+ del self.props[key]
+ #
+ # Write the has as a Java-like properties file
+ #
+ def write(self, propsfile):
+ f = open(propsfile, 'w')
+ items = self.props.items()
+ for (k, v) in items:
+ #print 'WRITING', k, '=', v
+ f.write(k + ' = ' + str(v) + '\n')
+ f.close()
+
+ #
+ # return a shallow copy of the dictionary
+ #
+ def copy_dictionary(self):
+ return self.props.copy()
+
+ #
+ # return the entries in the dictionary
+ #
+ def items(self):
+ return self.props.items()
+
+ #
+ # check to see if the key exists in the dictionary
+ #
+ def has_key(self, key):
+ return self.props.has_key(key)
+
+ #
+ # Return the length of the dictionary
+ #
+ def __len__(self):
+ return len(self.props)
+
+class DuccBase:
+
+
+ def read_properties(self):
+
+ self.ducc_properties = DuccProperties()
+ self.ducc_properties.load(self.propsfile)
+
+ self.webserver_node = self.ducc_properties.get('ducc.ws.node')
+ self.jvm = self.ducc_properties.get('ducc.jvm')
+
+ def java(self):
+ return self.jvm
+
+ def java_home(self):
+ if ( os.environ.has_key('DUCC_POST_INSTALL') ):
+ return 'JAVA_HOME' # avoid npe during first-time setup
+
+ if ( self.system == 'Darwin' ):
+ self.jvm_home = "/Library/Java/Home"
+ else:
+ ndx = self.jvm.rindex('/')
+ ndx = self.jvm.rindex('/', 0, ndx)
+ self.jvm_home = self.jvm[:ndx]
+
+ return self.jvm_home
+
+ def version(self):
+ lines = self.popen(self.jvm, ' org.apache.uima.ducc.utils.Version')
+ line = lines.readline().strip()
+ return "DUCC Version", line
+
+ # simply spawn-and-forget using Python preferred mechanism
+ def spawn(self, *CMD):
+ cmd = ' '.join(CMD)
+ # print '**** spawn', cmd, '****'
+ ducc = subprocess.Popen(cmd, shell=True)
+ pid = ducc.pid
+ status = os.waitpid(pid, 0)
+ return pid
+
+ def popen(self, *CMD):
+ cmd = ' '.join(CMD)
+ #print 'POPEN:', cmd
+ proc = subprocess.Popen(cmd, bufsize=0, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT)
+ return proc.stdout
+
+ def format_classpath(self, cp):
+ strings = cp.split(':')
+ for s in strings:
+ print s
+
+ def set_classpath(self):
+ ducc_home = self.DUCC_HOME
+ LIB = ducc_home + '/lib'
+
+ CLASSPATH = LIB + '/ducc-submit.jar'
+ os.environ['CLASSPATH'] = CLASSPATH
+
+ def which(self, file):
+ for p in os.environ["PATH"].split(":"):
+ if os.path.exists(p + "/" + file):
+ return p + "/" + file
+ return None
+
+ def mkargs(self, args):
+ '''
+ The cli needs to insure all args are fully quoted so the shell doesn't
+ lose the proper tokenization. This quotes everything.
+ '''
+ answer = []
+ for a in args:
+ arg = '"' + a + '"'
+ answer.append(arg)
+ return answer
+
+ def __init__(self):
+
+ if ( os.environ.has_key('DUCC_HOME') ):
+ self.DUCC_HOME = os.environ['DUCC_HOME']
+ else:
+ me = os.path.abspath(sys.argv[0])
+ ndx = me.rindex('/')
+ ndx = me.rindex('/', 0, ndx)
+ self.DUCC_HOME = me[:ndx] # split from 0 to ndx
+ os.environ['DUCC_HOME'] = self.DUCC_HOME
+
+
+ self.system = platform.system()
+ self.jvm = None
+ self.webserver_node = 'localhost'
+ self.propsfile = self.DUCC_HOME + '/resources/ducc.properties'
+ self.localhost = os.uname()[1]
+ self.read_properties()
+
+ os.environ['JAVA_HOME'] = self.java_home()
+
+ self.set_classpath()
+
+if __name__ == "__main__":
+ base = DuccBase()
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_base.py
------------------------------------------------------------------------------
svn:executable = *
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_cancel Mon Jul 8 19:29:04 2013
@@ -18,13 +18,10 @@
# under the License.
# -----------------------------------------------------------------------
+import sys
+from ducc_base import DuccBase
-from ducc_boot import *
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccCancel(DuccUtil):
+class DuccCancel(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-cancel.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_monitor Mon Jul 8 19:29:04 2013
@@ -18,13 +18,10 @@
# under the License.
# -----------------------------------------------------------------------
-
-from ducc_boot import *
-set_ducc_home()
+import sys
+from ducc_base import DuccBase
-from ducc_util import DuccUtil
-
-class DuccMonitor(DuccUtil):
+class DuccMonitor(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-monitor.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_perf_stats
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_perf_stats?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_perf_stats (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_perf_stats Mon Jul 8 19:29:04 2013
@@ -19,12 +19,10 @@
# -----------------------------------------------------------------------
-from ducc_boot import *
-set_ducc_home()
+import sys
+from ducc_base import DuccBase
-from ducc_util import DuccUtil
-
-class DuccPerfStats(DuccUtil):
+class DuccPerfStats(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-perf-stats.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_cancel
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_cancel?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_cancel (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_cancel Mon Jul 8 19:29:04 2013
@@ -19,13 +19,10 @@
# under the License.
# -----------------------------------------------------------------------
+import sys
+from ducc_base import DuccBase
-from ducc_boot import *
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccProcessCancel(DuccUtil):
+class DuccProcessCancel(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-process-cancel.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_submit
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_submit?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_submit (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_process_submit Mon Jul 8 19:29:04 2013
@@ -21,14 +21,9 @@
import sys
-import os
+from ducc_base import DuccBase
-from ducc_boot import *
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccProcessSubmit(DuccUtil):
+class DuccProcessSubmit(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-process-submit.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_reserve Mon Jul 8 19:29:04 2013
@@ -19,13 +19,10 @@
# under the License.
# -----------------------------------------------------------------------
+import sys
+from ducc_base import DuccBase
-from ducc_boot import *
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccReserve(DuccUtil):
+class DuccReserve(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-reserve.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_services
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_services?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_services (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_services Mon Jul 8 19:29:04 2013
@@ -19,15 +19,10 @@
# under the License.
# -----------------------------------------------------------------------
+import sys
+from ducc_base import DuccBase
-from ducc_boot import *
-import getopt
-
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccServices(DuccUtil):
+class DuccServices(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-services.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_submit Mon Jul 8 19:29:04 2013
@@ -21,14 +21,9 @@
import sys
-import os
+from ducc_base import DuccBase
-from ducc_boot import *
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccSubmit(DuccUtil):
+class DuccSubmit(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-submit.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_unreserve Mon Jul 8 19:29:04 2013
@@ -19,13 +19,10 @@
# under the License.
# -----------------------------------------------------------------------
+import sys
+from ducc_base import DuccBase
-from ducc_boot import *
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccUnReserve(DuccUtil):
+class DuccUnReserve(DuccBase):
def main(self, argv):
self.spawn(self.java(), '-jar', self.DUCC_HOME + '/lib/uima-ducc-unreserve.jar', ' '.join(self.mkargs(argv)))
Modified: uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_web
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_web?rev=1500896&r1=1500895&r2=1500896&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_web (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/scripts/ducc_web Mon Jul 8 19:29:04 2013
@@ -19,15 +19,10 @@
# under the License.
# -----------------------------------------------------------------------
+import sys
+from ducc_base import DuccBase
-from ducc_boot import *
-import getopt
-
-set_ducc_home()
-
-from ducc_util import DuccUtil
-
-class DuccWeb(DuccUtil):
+class DuccWeb(DuccBase):
def syntax(self):
print " --qm or --query-machines";