You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2016/03/18 23:32:05 UTC
svn commit: r1735698 -
/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install
Author: burn
Date: Fri Mar 18 22:32:05 2016
New Revision: 1735698
URL: http://svn.apache.org/viewvc?rev=1735698&view=rev
Log:
UIMA-4869 Script that installs or updates DUCC
Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install (with props)
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install?rev=1735698&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install Fri Mar 18 22:32:05 2016
@@ -0,0 +1,478 @@
+#! /usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+# -------------------------------------------------------------------------------------
+# Unpacks and installs DUCC, or updates an existing installation.
+# Updates in place so preserves all the site-specific files
+# Archives the current build in a timestamped direcory under DUCC_HOME/../ducc_archives
+# To revert back to an archived build:
+# - copy/move all of the archived files back to the runtime
+# -------------------------------------------------------------------------------------
+import os
+import sys
+import datetime
+import fnmatch
+import re
+import shutil
+
+def usage():
+ print "Usage: ducc_install ducc-binary-tarfile ducc-runtime"
+ print ""
+ print " Installs or updates DUCC at 'ducc_runtime' from the build in 'ducc-binary-tarfile'"
+ print " If there is no existing runtime a fresh installation will be placed in 'ducc_runtime'"
+ print " If the runtime exists, the update will:"
+ print " - check that DUCC is not running"
+ print " - create a site.ducc.properties file if updating from DUCC 1.1.0"
+ print " - create a time-stamped archive directory to hold the old runtime"
+ print " - archive and then update existing directories,"
+ print " - keep the directories: logs state history"
+ print " - keep any other files in directories: resources lib"
+ print " - keep the customized resources files: site.ducc.properties ducc.classes ducc.nodes jobdriver.nodes ducc.administrators"
+ print " - keep existing files in resources.private"
+ print " - keep the webserver/etc/keystore file"
+ print " - rebuild the non-privileged ducc_ling"
+ print " Note: any files added to directories other than resources & lib will not be retained, but will be archived"
+ print ""
+ print " To revert back to an archived build:"
+ print " - copy/move all of the archived files back to the runtime"
+
+#-----------------------------------------------------------------------------------------
+# Get the version of the build from the name of the cli jar
+#-----------------------------------------------------------------------------------------
+def get_oversion(runtime):
+ fns = fnmatch.filter(os.listdir(runtime + '/lib/uima-ducc'), 'uima-ducc-cli-*.jar')
+ if ( len(fns) == 0 ):
+ raise Exception("Not a valid DUCC installation - missing versioned cli jar")
+ m = re.match('uima\\-ducc\\-cli\\-([a-zA-Z0-9_\\.\\-]+)\\.jar', fns[0])
+ if ( m == None ):
+ raise Exception("Not a valid DUCC installation - invalid name: " + fns[0])
+ return m.group(1)
+
+#-----------------------------------------------------------------------------------------
+# Get the version of the build from the name of the tar file
+#-----------------------------------------------------------------------------------------
+def get_nversion(tarfile):
+ p = re.compile('^uima\\-ducc\\-([a-zA-Z0-9_\\.\\-]+)\\-bin\\.tar\\.gz$')
+ m = p.search(tarfile, 0)
+ if ( m == None ):
+ raise Exception("Invalid tar file name: " + tarfile + " ... expecting: uima-ducc-<version>-bin.tar.gz")
+ return m.group(1)
+
+#-----------------------------------------------------------------------------------------
+# Create archive directory
+#-----------------------------------------------------------------------------------------
+def create_archive(runtime, oversion, nversion):
+ runtimeParent,runtimeName = os.path.split(runtime)
+ now = datetime.datetime.now()
+ nowstr = now.strftime('%Y%m%d-%H%M')
+ archiveParent = os.path.join(runtimeParent, 'ducc_archives')
+ archive = os.path.join(archiveParent, runtimeName + '_' + nowstr)
+ if not os.path.exists(archive):
+ os.makedirs(archive)
+
+ print " --- Updating DUCC to", nversion, " and archiving", oversion, "to", archive
+ print " NOTE: this update may be reversed by copying back all of the archived files,"
+ print " e.g. cp --recursive --remove-destination", archive+"/*", runtime
+
+ return archive
+
+
+#-----------------------------------------------------------------------------------------
+# Clear out the old expanded tarball if needed, and expand the new one
+#-----------------------------------------------------------------------------------------
+def expand_tarball(tarfile, nversion, outdir):
+ print "\n", " --- Expanding tar file"
+ extract = os.path.join(outdir, 'apache-uima-ducc-' + nversion)
+ try:
+ if ( os.path.exists(extract) ):
+ shutil.rmtree(extract);
+ except:
+ raise Exception("Cannot remove old tarball extract: " + extract)
+
+ cmd = 'umask 022 && tar -C ' + outdir + ' -xf ' + tarfile
+ rc = os.system(cmd);
+ if ( rc != 0 ):
+ raise Exception("Command fails, rc= " + str(rc) + "\n " + cmd)
+
+ if not os.path.exists(extract):
+ raise Exception("Cannot extract runtime from tarball " + tarfile + " Expecting to find " + extract)
+
+ return extract
+
+#-----------------------------------------------------------------------------------------
+# Convert final comments into a DUCC 1.1.0 properties file
+#-----------------------------------------------------------------------------------------
+def create_110_properties(source, dest):
+ found = False
+ with open(source) as f:
+ for line in f:
+ if not found:
+ if line.startswith("#=====MARKER====="):
+ found = True
+ outf = open(dest, 'w')
+ else:
+ outf.write('ducc.' + line[1:])
+ outf.close();
+
+#-----------------------------------------------------------------------------------------
+# Update directory by archiving all files that are replaced, unless in exclude list
+# Optionally rename not-replaced files that differ from the existing files
+#-----------------------------------------------------------------------------------------
+def update_dir(subdir, excludeFiles, rename):
+ olddir = os.path.join(runtime, subdir)
+ newdir = os.path.join(newducc, subdir)
+ archdir = os.path.join(archive, subdir)
+
+ if not os.path.exists(archdir):
+ os.mkdir(archdir)
+
+ files = os.listdir(newdir)
+ for f in files:
+ curf = os.path.join(olddir, f)
+ srcf = os.path.join(newdir, f)
+ if f in excludeFiles or excludeFiles == '*':
+ if os.path.exists(curf):
+ if rename and os.path.isfile(curf):
+ cmd = 'cmp --quiet ' + srcf + ' ' + curf
+ rc = os.system(cmd);
+ if rc != 0:
+ os.rename(srcf, curf + '-new')
+ print "Keeping", f, '(new version saved as', f+'-new)'
+ else:
+ print "Adding", resf
+ os.rename(srcf, curf)
+ else:
+ if os.path.exists(curf):
+ print "Replacing", f
+ os.rename(curf, os.path.join(archdir,f))
+ else:
+ print "Adding", f
+ os.rename(srcf, curf)
+
+#-----------------------------------------------------------------------------------------
+# Replace the shipped place-holder files with the local ones
+# (Could check 1st for differences)
+#-----------------------------------------------------------------------------------------
+def replace_place_holders(files):
+ for f in files:
+ print "\n", " --- Restoring original ", f
+ os.rename( os.path.join(archive, f), os.path.join(runtime, f) )
+
+
+#=========================================================================================
+# Main program:
+# tarfile runtime
+#=========================================================================================
+
+if len(sys.argv) != 3:
+ usage()
+ exit(1)
+
+tarfile = sys.argv[1]
+rtime = sys.argv[2]
+runtime = os.path.realpath(rtime)
+
+if not os.path.exists(tarfile):
+ print "ERROR - Missing tar file", tarfile
+ exit(1)
+nversion = get_nversion(os.path.basename(tarfile))
+
+#-----------------------------------------------------------------------------------------
+# Check if is a new install ... just untar
+#-----------------------------------------------------------------------------------------
+if not os.path.exists(runtime):
+ print " --- A fresh installation of DUCC", nversion, "will be created at", runtime
+ runtimeParent = os.path.dirname(runtime)
+ if not os.path.exists(runtimeParent):
+ os.makedirs(runtimeParent)
+ newducc = expand_tarball( tarfile, nversion, runtimeParent )
+ os.rename(newducc, runtime)
+ print "\n", " --- Installation completed ... please run 'admin/ducc_post_install'"
+ exit(0)
+
+#-----------------------------------------------------------------------------------------
+# Check if appears to be a valid (stopped) DUCC installation
+#-----------------------------------------------------------------------------------------
+if not os.path.exists(os.path.join(runtime, 'resources/ducc.properties')):
+ print "ERROR - Not a valid DUCC runtime directory:", runtime
+ exit(1)
+if os.path.exists(os.path.join(runtime, 'state/ducc.pids')):
+ print "ERROR - DUCC appears to be running ... please run '" + rtime + "/admin/stop_ducc -a'"
+ exit(1)
+
+oversion = get_oversion(runtime)
+if oversion == '1.0.0':
+ print "Sorry, migration not supported for DUCC 1.0.0 at present"
+ exit(9)
+
+#-----------------------------------------------------------------------------------------
+# Create a time-stamped archive and expand the tarball into its parent
+#-----------------------------------------------------------------------------------------
+archive = create_archive(runtime, oversion, nversion)
+newducc = expand_tarball( tarfile, nversion, os.path.dirname(archive) )
+
+#-----------------------------------------------------------------------------------------
+# May need to create the new webserver directory for 2.0
+#-----------------------------------------------------------------------------------------
+weblogdir = os.path.join(runtime, 'logs/webserver')
+if not os.path.exists(weblogdir):
+ os.makedirs(weblogdir)
+
+#-----------------------------------------------------------------------------------------
+# Create a site.ducc.properties file if missing ... only for DUCC 1.1.0
+#-----------------------------------------------------------------------------------------
+siteProps = os.path.join(runtime, 'resources/site.ducc.properties')
+if not os.path.exists(siteProps):
+ if oversion != '1.1.0':
+ print "Missing site.ducc.properties - can only be created for 1.1.0"
+ exit(9)
+ currentProps = os.path.join(runtime, 'resources/ducc.properties')
+ originalProps = os.path.join(runtime, 'resources/ducc-1.1.0.properties')
+ create_110_properties(os.path.realpath(sys.argv[0]), originalProps)
+
+ if not os.path.exists(originalProps):
+ print "ERROR - Failed to create the 1.1.0 properties file from the ending comments in this script"
+ exit(9)
+
+ # Use the new props manager - use abs fnames as otherwise are relative to the deduced DUCC_HOME/resources
+ cmd = newducc + '/admin/ducc_props_manager --delta ' + originalProps + ' --with ' + currentProps + ' --to ' + siteProps
+ rc = os.system(cmd)
+ if rc != 0:
+ print "ERROR", rc, "Failed to create", siteProps
+ exit(9)
+ print " --- Created a file with just the site-specific properties:", siteProps
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) all directories in the new build EXCEPT resources & lib
+# Note that the history, logs, & state directories are not part of a build so are left unchanged
+#-----------------------------------------------------------------------------------------
+print "\n", " --- Processing", os.path.basename(runtime), "folder:"
+update_dir('', [ 'lib', 'resources', 'resources.private' ], False)
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) all files in the lib directory
+# This ensures that any site local jars remain
+#-----------------------------------------------------------------------------------------
+print "\n", " --- Processing lib folder:"
+update_dir('lib', [], False)
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) some of the files in resources
+# Don't change any that are site-specific
+# (The nodes files should not be in the new build, but just in case ...)
+#-----------------------------------------------------------------------------------------
+print "\n", " --- Processing resources folder:"
+resarchive = os.path.join(archive, 'resources')
+preserveFiles = [ 'ducc.classes', 'ducc.administrators', 'ducc.nodes', 'jobdriver.nodes', 'service_monitors' ]
+update_dir('resources', preserveFiles, True)
+
+#-----------------------------------------------------------------------------------------
+# Add but don't replace any existing files in resources.private
+# Rename any files that differ from the existing ones
+#-----------------------------------------------------------------------------------------
+print "\n", " --- Processing resources.private folder:"
+update_dir('resources.private', '*', True)
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) all files in resources/service_monitors
+# This ensures that any local registered pingers remain
+#-----------------------------------------------------------------------------------------
+print "\n", " --- Processing resources/service_monitors folder:"
+update_dir('resources/service_monitors', [], False)
+
+#-----------------------------------------------------------------------------------------
+# Restore a few site-modified files from the archive
+# - the local_hooks.py may define local versions of 3 utility functions:
+# find_other_processes, verify_slave_node, verify_master_node
+# - the keystore is created by ducc-post-install using the password in resources.private
+# - the site.jsp may define extra ducc-mon buttons
+# - the ducc.local.js may define local versions of 2 place-holder functions:
+# ducc_init_local, ducc_update_page_local
+#-----------------------------------------------------------------------------------------
+replaceFiles = [ 'admin/local_hooks.py', 'webserver/etc/keystore', 'webserver/root/site.jsp', 'webserver/root/js/ducc.local.js' ]
+replace_place_holders(replaceFiles)
+
+#-----------------------------------------------------------------------------------------
+# Delete what's left of the extract (just the resources & lib folders)
+#-----------------------------------------------------------------------------------------
+shutil.rmtree(newducc)
+
+#-----------------------------------------------------------------------------------------
+# Re-build ducc_ling
+# Since it needs ducc.properties run the merge from the admin directory
+#-----------------------------------------------------------------------------------------
+print "\n", " --- Rebuilding ducc_ling"
+os.chdir(runtime + '/admin')
+rc = os.system('./ducc_props_manager --merge ../resources/default.ducc.properties --with ../resources/site.ducc.properties --to ../resources/ducc.properties')
+if (rc != 0):
+ print "ERROR - failed to create ducc.properties and to rebuild ducc_ling"
+ exit(9)
+rc = os.system('./build_duccling')
+if (rc != 0):
+ print "ERROR - failed to rebuild ducc_ling"
+ exit(9)
+
+print "\n", " --- Installation completed!"
+print " NOTE - if your ducc_ling is privileged you should update it"
+
+#->->->->->->-> DO NOT CHANGE ANYTHING BELOW THIS MARKER <-<-<-<-<-<-<-
+#=====MARKER===== The following are the original ducc.properties shipped with DUCC 1.1.0
+#head=<head-node>
+#jvm=<full-path-to-java-command>
+#cluster.name=Apache UIMA-DUCC
+#private.resources=${DUCC_HOME}/resources.private
+#jms.provider=activemq
+#broker.protocol=tcp
+#broker.hostname=${ducc.head}
+#broker.port=61617
+#broker.url.decoration=jms.useCompression=true
+#broker.name=localhost
+#broker.jmx.port=1100
+#broker.credentials.file=${ducc.private.resources}/ducc-broker-credentials.properties
+#broker.automanage=true
+#broker.memory.options=-Xmx1G
+#broker.configuration=conf/activemq-ducc.xml
+#broker.home=${DUCC_HOME}/apache-uima/apache-activemq
+#broker.server.url.decoration=transport.soWriteTimeout=45000
+#locale.language=en
+#locale.country=us
+#node.min.swap.threshold=0
+#admin.endpoint=ducc.admin.channel
+#admin.endpoint.type=topic
+#jmx.port=2099
+#agent.jvm.args=-Xmx500M
+#orchestrator.jvm.args=-Xmx1G
+#rm.jvm.args=-Xmx1G
+#pm.jvm.args=-Xmx1G
+#sm.jvm.args=-Xmx1G
+#db.jvm.args=-Xmx2G
+#ws.jvm.args=-Xmx2G -Djava.util.Arrays.useLegacyMergeSort=true
+#environment.propagated=USER HOME LANG
+#cli.httpclient.sotimeout=0
+#signature.required=on
+#db.configuration.class=org.apache.uima.ducc.db.config.DbComponentConfiguration
+#db.state.update.endpoint=ducc.db.state
+#db.state.update.endpoint.type=topic
+#db.state.publish.rate=15000
+#ws.configuration.class=org.apache.uima.ducc.ws.config.WebServerConfiguration
+#ws.port=42133
+#ws.port.ssl=42155
+#ws.session.minutes=60
+#ws.automatic.cancel.minutes=5
+#ws.max.history.entries=4096
+#ws.jsp.compilation.directory=/tmp/ducc/jsp
+#ws.login.enabled=false
+#ws.visualization.strip.domain=true
+#jd.configuration.class=org.apache.uima.ducc.jd.config.JobDriverConfiguration
+#jd.state.update.endpoint=ducc.jd.state
+#jd.state.update.endpoint.type=topic
+#jd.state.publish.rate=15000
+#jd.queue.prefix=ducc.jd.queue.
+#jd.queue.timeout.minutes=5
+#jd.host.class=JobDriver
+#jd.host.description=Job Driver
+#jd.host.memory.size=2GB
+#jd.host.number.of.machines=1
+#jd.host.user=System
+#jd.share.quantum=400
+#threads.limit=500
+#driver.jvm.args=-Xmx300M
+#sm.configuration.class=org.apache.uima.ducc.sm.config.ServiceManagerConfiguration
+#sm.state.update.endpoint=ducc.sm.state
+#sm.state.update.endpoint.type=topic
+#sm.default.monitor.class=org.apache.uima.ducc.cli.UimaAsPing
+#sm.instance.failure.max=5
+#sm.instance.failure.limit=${ducc.sm.instance.failure.max}
+#sm.instance.failure.window=30
+#sm.init.failure.limit=1
+#sm.meta.ping.rate=60000
+#sm.meta.ping.stability=10
+#sm.meta.ping.timeout=15000
+#sm.http.port=19989
+#sm.http.node=${ducc.head}
+#sm.default.linger=300000
+#orchestrator.configuration.class=org.apache.uima.ducc.orchestrator.config.OrchestratorConfiguration
+#orchestrator.start.type=warm
+#orchestrator.state.update.endpoint=ducc.orchestrator.state
+#orchestrator.state.update.endpoint.type=topic
+#orchestrator.state.publish.rate=10000
+#orchestrator.abbreviated.state.update.endpoint=ducc.orchestrator.abbreviated.state
+#orchestrator.abbreviated.state.update.endpoint.type=topic
+#orchestrator.abbreviated.state.publish.rate=10000
+#orchestrator.maintenance.rate=60000
+#orchestrator.http.port=19988
+#orchestrator.http.node=${ducc.head}
+#orchestrator.unmanaged.reservations.accepted=true
+#rm.configuration.class=org.apache.uima.ducc.rm.config.ResourceManagerConfiguration
+#rm.state.update.endpoint=ducc.rm.state
+#rm.state.update.endpoint.type=topic
+#rm.state.publish.rate=10000
+#rm.share.quantum=1
+#rm.scheduler=org.apache.uima.ducc.rm.scheduler.NodepoolScheduler
+#rm.class.definitions=ducc.classes
+#rm.default.memory=4
+#rm.init.stability=2
+#rm.node.stability=5
+#rm.eviction.policy=SHRINK_BY_INVESTMENT
+#rm.initialization.cap=1
+#rm.expand.by.doubling=true
+#rm.prediction=true
+#rm.prediction.fudge=120000
+#rm.fragmentation.threshold=8
+#rm.admin.endpoint=ducc.rm.admin.channel
+#rm.admin.endpoint.type=queue
+#agent.configuration.class=org.apache.uima.ducc.agent.config.AgentConfiguration
+#agent.request.endpoint=ducc.agent
+#agent.request.endpoint.type=topic
+#agent.managed.process.state.update.endpoint=ducc.managed.process.state.update
+#agent.managed.process.state.update.endpoint.type=socket
+#agent.managed.process.state.update.endpoint.params=transferExchange=true&sync=false
+#agent.node.metrics.sys.gid.max=500
+#agent.node.metrics.publish.rate=30000
+#agent.node.metrics.endpoint=ducc.node.metrics
+#agent.node.metrics.endpoint.type=topic
+#agent.node.inventory.publish.rate=10000
+#agent.node.inventory.publish.rate.skip=30
+#agent.node.inventory.endpoint=ducc.node.inventory
+#agent.node.inventory.endpoint.type=topic
+#agent.launcher.thread.pool.size=10
+#agent.launcher.use.ducc_spawn=true
+#agent.launcher.ducc_spawn_path=${DUCC_HOME}/admin/ducc_ling
+#agent.launcher.process.stop.timeout=60000
+#agent.launcher.process.init.timeout=7200000
+#agent.rogue.process.user.exclusion.filter=
+#agent.rogue.process.exclusion.filter=sshd:,-bash,-sh,/bin/sh,/bin/bash,grep,ps
+#agent.share.size.fudge.factor=5
+#agent.launcher.cgroups.enable=false
+#agent.launcher.cgroups.utils.dir=/usr/bin,/bin
+#agent.exclusion.file=${DUCC_HOME}/resources/exclusion.nodes
+#pm.configuration.class=org.apache.uima.ducc.pm.config.ProcessManagerConfiguration
+#pm.request.endpoint=ducc.pm
+#pm.request.endpoint.type=queue
+#pm.state.update.endpoint=ducc.pm.state
+#pm.state.update.endpoint.type=topic
+#pm.state.publish.rate=15000
+#uima-as.configuration.class=org.apache.uima.ducc.agent.deploy.uima.UimaAsServiceConfiguration
+#uima-as.endpoint=ducc.job.managed.service
+#uima-as.endpoint.type=socket
+#uima-as.endpoint.params=transferExchange=true&sync=false
+#uima-as.saxon.jar.path=file:${DUCC_HOME}/apache-uima/saxon/saxon8.jar
+#uima-as.dd2spring.xsl.path=${DUCC_HOME}/apache-uima/bin/dd2spring.xsl
+#flow-controller.specifier=org.apache.uima.ducc.common.uima.DuccJobProcessFC
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_install
------------------------------------------------------------------------------
svn:executable = *