You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/05/05 20:27:36 UTC
svn commit: r1677876 - in /uima/sandbox/uima-ducc/trunk/src/main/admin:
check_ducc ducc.py ducc_post_install ducc_util.py start_ducc
Author: challngr
Date: Tue May 5 18:27:36 2015
New Revision: 1677876
URL: http://svn.apache.org/r1677876
Log:
UIMA-4358 Remove single-user mode. Do permission checking in post-install and abort if it looks wrong.
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1677876&r1=1677875&r2=1677876&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue May 5 18:27:36 2015
@@ -44,7 +44,11 @@ class CheckDucc(DuccUtil):
self.check_clock_skew(checkdate)
self.verify_jvm()
self.verify_limits()
- self.verify_duccling(self.single_user)
+ (viable, elevated, safe) = self.verify_duccling()
+ self.duccling_ok(viable, elevated, safe)
+ if ( not safe or not viable ):
+ print 'NOTOK ducc_ling is not installed correctly.'
+
return
def verify_activemq(self):
@@ -103,10 +107,7 @@ class CheckDucc(DuccUtil):
if ( self.kill_signal == None ):
response = "Node health checks return."
- if ( self.single_user ) :
- lines = self.ssh(node, True, self.DUCC_HOME + "/admin/check_ducc", "-s", "-x", str(int(time())))
- else:
- lines = self.ssh(node, True, self.DUCC_HOME + "/admin/check_ducc", "-x", str(int(time())))
+ lines = self.ssh(node, True, self.DUCC_HOME + "/admin/check_ducc", "-x", str(int(time())))
while 1:
line = lines.readline()
if ( 'signal' in line ):
@@ -171,9 +172,6 @@ class CheckDucc(DuccUtil):
print " the PID file needs rebuilding. This option causes the file to be rebuilt regardless of"
print " changes."
print ""
- print " -s --singleuser"
- print " Bypasses the multi-user ducc_ling checks."
- print ""
print " -x localdate"
print " Validate the local installation, called via ssh usually. The date is the dat on the calling machine."
print ""
@@ -189,7 +187,7 @@ class CheckDucc(DuccUtil):
def main(self, argv):
try:
- opts, args = getopt.getopt(argv, 'cikn:opqsx:h?v', ['configuration', 'nodelist=', 'int', 'quit', 'kill', 'pids', 'verbose', 'nothreading', 'singleuser'])
+ opts, args = getopt.getopt(argv, 'cikn:opqx:h?v', ['configuration', 'nodelist=', 'int', 'quit', 'kill', 'pids', 'verbose', 'nothreading', ])
except:
self.usage("Invalid arguments " + ' '.join(argv))
@@ -202,7 +200,6 @@ class CheckDucc(DuccUtil):
checkdate = 0
config_only = False
verbose = False
- self.single_user = False
for ( o, a ) in opts:
if o in ('-c', '--configuration'):
@@ -226,8 +223,6 @@ class CheckDucc(DuccUtil):
self.kill_signal = '-KILL'
elif o in ( '--nothreading' ):
self.disable_threading()
- elif o in ('-s', '--singleuser' ):
- self.single_user = True
elif o in ('-p', '--pids'):
redo_pids = True
elif o in ('-x'):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1677876&r1=1677875&r2=1677876&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue May 5 18:27:36 2015
@@ -65,7 +65,7 @@ class Ducc(DuccUtil):
def add_to_classpath(self, lib):
os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ":" + lib
- def run_component(self, component, or_parms, numagents, rmoverride, background, nodup, localdate, single_user):
+ def run_component(self, component, or_parms, numagents, rmoverride, background, nodup, localdate):
if ( component == 'all' ):
component = 'rm,sm,pm,ws,orchestrator'
@@ -101,14 +101,10 @@ class Ducc(DuccUtil):
if ( not self.verify_limits() ):
return
- if ( not single_user ) :
- dok = self.verify_duccling(single_user)
- if ( not dok ):
- print 'NOTOK ducc_ling is not set up correctly on node', self.localhost
- print dok
- return
- else:
- print 'Single user mode: bypassing ducc_ling checks.'
+ (viable, elevated, safe) = self.verify_duccling()
+ if ( not self.duccling_ok(viable, elevated, safe) ):
+ print 'NOT_OK Cannot proceed because of ducc_ling problems.'
+ return
if ( not verify_slave_node(localdate, self.ducc_properties) ):
# we assume that verify_local_node is spewing a line of the form
@@ -219,8 +215,7 @@ class Ducc(DuccUtil):
pid = self.nohup(cmd)
else:
pid = self.spawn(' '.join(cmd))
- print 'PID ' + str(pid) # nohup will print this from the (twice) forked process if background
- # hard for us to access it here in nohup
+ print 'PID ' + str(pid)
if ( c == 'ws' ):
os.chdir(here)
@@ -243,7 +238,6 @@ class Ducc(DuccUtil):
print ' -n <numagents> if > 1, multiple agents are started (testing mode)'
print ' -o <mem-in-GB> rm memory override for use on small machines'
print ' -k causes the entire DUCC system to shutdown'
- print ' -s start in single-user mode (inhibit some sanity checks)'
print ' --nodup If specified, do not start a process if it appears to be already started.'
print ' --or_parms [cold|warm|hot]'
print ' --ducc_head nodename the name of the "ducc head" where ducc is started from'
@@ -259,7 +253,6 @@ class Ducc(DuccUtil):
shutdown = False
background = False
or_parms = None
- single_user = False
nodup = False # we allow duplicates unless asked not to
localdate = time.time()
@@ -283,8 +276,6 @@ class Ducc(DuccUtil):
rmoverride = a
elif ( o == '-k'):
shutdown = True
- elif ( o == '-s'):
- single_user = True
elif ( o == '--or_parms' ):
or_parms = a
elif ( o == '--nodup' ):
@@ -308,7 +299,7 @@ class Ducc(DuccUtil):
if ( component == 'broker' ):
self.run_broker(background)
else:
- self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate, single_user)
+ self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate)
return
def __call__(self, *args):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1677876&r1=1677875&r2=1677876&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Tue May 5 18:27:36 2015
@@ -25,6 +25,7 @@ import getopt
import shutil
import subprocess
+from stat import *
from ducc_util import DuccUtil
from ducc_base import Properties
@@ -243,8 +244,69 @@ class PostInstall():
shutil.move(fn, bak)
print 'Existing', fn, 'moved to', bak
+ def verify_permissions(self):
+ # should have 755 permissions
+ spot_checked_directories = ['../bin', '../lib', '../resources' ]
+ # should have 755 permissions
+ spot_checked_execs = ['../bin/ducc_submit']
+ # should have 644 permissions
+ spot_checked_data = ['../lib/uima-ducc-cli.jar', '../resources/default.ducc.properties']
+
+ ret = True
+ for f in spot_checked_directories:
+ if ( not os.path.exists(f) ):
+ print 'ERROR: Directory', f, 'cannot be found.'
+ ret = false
+ continue
+
+ stat = os.stat(f)
+ mode = oct(stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
+ expected = oct(0755)
+ if ( mode != expected ):
+ print 'ERROR: Directory', f, 'has permissions', mode, 'expected', expected
+ ret = False
+
+ for f in spot_checked_execs:
+ if ( not os.path.exists(f) ):
+ print 'ERROR: File', f, 'cannot be found.'
+ ret = False
+ continue
+
+ stat = os.stat(f)
+ mode = oct(stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
+ expected = oct(0755)
+ if ( mode != expected ):
+ print 'ERROR: File', f, 'has permissions', mode, 'expected', expected
+ ret = False
+
+ for f in spot_checked_data:
+ if ( not os.path.exists(f) ):
+ print 'ERROR: File', f, 'cannot be found.'
+ ret = False
+ continue
+
+ stat = os.stat(f)
+ mode = oct(stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
+ expected = oct(0644)
+ if ( mode != expected ):
+ print 'ERROR: File', f, 'has permissions', mode, 'expected', expected
+ ret = False
+
+ return ret
+
def main(self, argv):
+ if ( not self.verify_permissions() ):
+ print '--------------------------------------------------------------------------------'
+ print 'Package verificaiton fails. Most likely cause is an unexpected UMASK unpacking the distribution.'
+ print 'To unpack the distribution your UMASK must be set to 022.'
+ print ''
+ print 'Example:'
+ print ''
+ print 'umask 022; tar -xf [distribution]'
+ print '--------------------------------------------------------------------------------'
+ sys.exit(1)
+
self.ducc_head = None
self.keystore_pw = None
self.path_to_java = None
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1677876&r1=1677875&r2=1677876&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue May 5 18:27:36 2015
@@ -338,59 +338,89 @@ class DuccUtil(DuccBase):
return False
return True
- def verify_duccling(self, single_user):
+ #
+ # Verify the viability of ducc_ling.
+ # Returns a tuple (viable, elevated, safe)
+ # where 'viable' is a boolean indicating whether the ducc_ling is viable (exists and is correct version)
+ # If this is false, the other two values are meaningless
+ #
+ # 'elevated' indicates whether priveleges are at least partially elevated
+ # 'safe' indicates whether ducc_ling is safe (elevated privileges and correct permissions)
+ # The caller will evaluate these and take appriopriate action
+ #
+ def verify_duccling(self):
+ viable = True
+ elevated = False
+ safe = False
+
+ dl = self.duccling
+ path = os.path.dirname(os.path.abspath(dl))
+
- check_permission = True # if we're not ducc we don't care about permissions
- user = os.environ['LOGNAME']
- if ( (user != 'ducc') or ( single_user) ):
- check_permission = False
-
- if ( check_permission ) : # only care about ducc_ling setup if we're ducc
- path = os.path.dirname(os.path.abspath(self.duccling))
- dl = path + '/ducc_ling'
-
- sstat = os.stat(path)
- mode = sstat.st_mode
- if ( not S_ISDIR(mode) ):
- print 'ducc_ling path', path, ': Not a directory.'
- return False
-
- dirown = mode & (S_IRWXU | S_IRWXG | S_IRWXO)
- #print 'Directory perms', oct(dirown)
- if ( dirown != S_IRWXU ):
- print 'ducc_ling path', path, ': Invalid directory permissions', oct(dirown), 'should be', oct(S_IRWXU)
- return False
-
- sstat = os.stat(dl)
- mode = sstat.st_mode
- expected = (S_IRWXU | S_IRGRP | S_IXGRP)
- pathown = mode & (S_IRWXU | S_IRWXG | S_IRWXO)
- #print 'Duccling perms', oct(pathown)
- if ( pathown != expected ):
- print 'ducc_ling module', dl, ': Invalid permissions', oct(pathown), 'Should be', oct(expected)
- return False
-
- if ( (mode & S_ISUID) != S_ISUID):
- print 'ducc_ling module', dl, ': setuid bit is not set'
- return False
-
+ if ( not (os.path.exists(dl) and os.access(dl, os.X_OK)) ):
+ print dl, 'does not exist or is not executable.'
+ viable = False
+
+ path = os.path.dirname(os.path.abspath(dl))
+ dl = path + '/ducc_ling'
+
+ dl_stat = os.stat(dl) # dl_stat is stat for ducc_ling
+ dl_mode = dl_stat.st_mode
+
+ if ( (dl_mode & S_ISUID) != S_ISUID):
+ if ( os.environ['LOGNAME'] == 'ducc' ):
+ print 'ducc_ling module', dl, ': setuid bit is not set. Processes will run as user ducc'
+ elevated = False
+ file_safe = True
+ dir_safe = True
+ own_safe = True
+ else:
+ elevated = True
+ file_safe = False
+ dir_safe = False
+ own_safe = False
+
+
+ if ( elevated ):
+
+ #
+ # if setuid bit is set, all this MUST be true or we won't mark ducc_ling safe:
+ # file permissions are 750
+ # dir permissions are 700
+ # owenership is root.ducc
+ #
+ dl_perm = oct(dl_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
+ expected = oct(0750)
+ if ( dl_perm != expected ):
+ print dl, ': Invalid execution bits', dl_perm, 'should be', expected
+ else:
+ file_safe = True
+
+ dir_stat = os.stat(path) # dir_stat is stat for ducc_ling
+ dir_mode = dir_stat.st_mode
+
+ dir_perm = oct(dir_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
+ expected = oct(0700)
+ if ( dir_perm != expected ):
+ print 'ducc_ling', path, ': Invalid directory permissions', dir_perm, 'should be', expected
+ else:
+ dir_safe = True
+
try:
grpinfo = grp.getgrnam('ducc')
+ duccgid = grpinfo.gr_gid
+
+ if ( (dl_stat.st_uid != 0) or (dl_stat.st_gid != duccgid) ):
+ print 'ducc_ling module', dl, ': Invalid ownership. Should be root.ducc'
+ else:
+ own_safe = True
except:
print 'ducc_ling group "ducc" cannot be found.'
- return False
- duccgid = grpinfo.gr_gid
- #print 'UID', sstat.st_uid, 'GID', duccgid
- if ( (sstat.st_uid != 0) or (sstat.st_gid != duccgid) ):
- print 'ducc_ling module', dl, ': Invalid ownership. Should be root.ducc'
- return False
- else:
- if ( not os.path.exists(self.duccling) ):
- print "Missing ducc_ling"
- return False
-
- # now make sure the version matches that on the master node
+ safe = file_safe and dir_safe and own_safe
+
+ # A last viability check, do versions match? This also runs it, proving it
+ # can execute in this environment.
lines = self.popen(self.duccling + ' -v')
version_from_head = lines.readline().strip();
toks = version_from_head.split()
@@ -407,13 +437,40 @@ class DuccUtil(DuccBase):
print "Mismatched ducc_ling versions:"
print "ALERT: Version on Agent Node:", version_from_head
print "ALERT: Version on Ducc Head:", line
- return False
+ viable = False
verfile.close()
else:
- print "ducc_ling version file missing, cannot verify version."
- return False;
+ print "NOTE: ducc_ling version file missing, cannot verify version."
+
+ # leave the decisions to the caller
+ return (viable, elevated, safe)
+
+ # Apply these rules to determine if ducc_ling is installed ok
+ #
+ # Caller Elevated Protected (safe) Action
+ # -------- -------- --------- ------
+ # ducc Y Y OK
+ # Y N Fail
+ #
+ # N Y (by def) OK
+ #
+ # ~ducc Y N (by def) Fail
+ # N Y (by def) OK, Note
+ def duccling_ok(self, viable, elevated, safe):
+
+ if ( not viable ):
+ return False
+
+ user = os.environ['LOGNAME']
+
+ if ( user == 'ducc' ):
+ if ( elevated ):
+ return safe
+ else:
+ if ( elevated ):
+ return False
+ print 'Note: Running unprivileged ducc_ling. Process will run as user', user
- print 'ducc_ling OK'
return True
def ssh_ok(self, node, line):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1677876&r1=1677875&r2=1677876&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue May 5 18:27:36 2015
@@ -63,7 +63,7 @@ class StartDucc(DuccUtil):
def start_component(self, args):
- ducc, component, or_parms, single_user = args
+ ducc, component, or_parms = args
msgs = []
node = self.ducc_properties.get('ducc.head')
@@ -91,10 +91,7 @@ class StartDucc(DuccUtil):
if ( node == 'local' ):
node = self.localhost
- if ( single_user ):
- lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-s', '-b', or_parms, '-d', str(time.time()), '--nodup', "'")
- else:
- lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-b', or_parms, '-d', str(time.time()), '--nodup', "'")
+ lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-b', or_parms, '-d', str(time.time()), '--nodup', "'")
# we'll capture anything that the python shell spews because it may be useful, and then drop the
# pipe when we see a PID message
@@ -121,18 +118,15 @@ class StartDucc(DuccUtil):
def start_one_agent(self, args):
- host, single_user = args
+ host = args[0]
msgs = []
spacer = ' '
msgs.append((host, ""))
- if (single_user):
- lines = self.ssh(host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c' 'agent', '-s', '-b', '-d', str(time.time()), '--nodup', "'")
- else:
- lines = self.ssh(host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c' 'agent', '-b', '-d', str(time.time()), '--nodup', "'")
-
- while 1:
- line = lines.readline().strip()
- #msgs.append(('[l]', line))
+ lines = self.ssh(host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c' 'agent', '-b', '-d', str(time.time()), '--nodup', "'")
+ for line in lines:
+ line = line.strip()
+ # print '[]', host, line
+ # msgs.append(('[l]', line))
if ( line.startswith('PID') ):
toks = line.split(' ')
pid = toks[1]
@@ -144,12 +138,16 @@ class StartDucc(DuccUtil):
msgs.append((spacer, 'DUCC Agent started PID', pid))
break
- if ( not line ):
- break
+ if ( 'tty' in line ):
+ # ssh junk if mesg is set
+ continue
+
toks = line.split()
- if ( not self.ssh_ok(host, line ) ):
- break;
+ sshmsgs = self.ssh_ok(host, line )
+ if ( sshmsgs != None ):
+ for m in sshmsgs:
+ print '[S]', m
if ( toks[0] == 'NOTOK' ):
msgs.append((spacer, 'NOTOK Not started:', ' '.join(toks[1:])))
@@ -180,10 +178,6 @@ class StartDucc(DuccUtil):
print ""
print " start_ducc -n foo.nodes -n bar.nodes -n baz.nodes"
print ""
- print " -s --singleuser"
- print " Start ducc in 'single user mode'. This bypasses some checking required for multi-user"
- print " mode and not required for single-user mode."
- print ""
print " -c, --component component"
print " Start a specific DUCC component, optionally on a specific node. If the component name"
print " is qualified with a nodename, the component is started on that node. To qualify a"
@@ -233,13 +227,12 @@ class StartDucc(DuccUtil):
nodefiles = []
components = []
- single_user = False
or_parms = self.ducc_properties.get('ducc.orchestrator.start.type')
self.pids = Properties()
self.pids.load_if_exists(self.pid_file)
try:
- opts, args = getopt.getopt(argv, 'c:mn:sh?v', ['component=', 'components=', 'help', 'nodelist=', 'singleuser', 'cold', 'warm', 'hot', 'nothreading'])
+ opts, args = getopt.getopt(argv, 'c:mn:sh?v', ['component=', 'components=', 'help', 'nodelist=', 'cold', 'warm', 'hot', 'nothreading'])
except:
self.invalid('Invalid arguments', ' '.join(argv))
@@ -248,8 +241,6 @@ class StartDucc(DuccUtil):
components.append(a)
elif o in ( '-n', '--nodelist' ):
nodefiles.append(a)
- elif o in ( '-s', '--singleuser' ):
- single_user = True
elif o in ( '--nothreading' ):
self.disable_threading()
elif o in ( '--cold', '--warm', '--hot' ):
@@ -335,7 +326,7 @@ class StartDucc(DuccUtil):
print '********** Starting agents from file', nodefile
try:
for node in nodelist:
- self.threadpool.invoke(self.start_one_agent, node, single_user)
+ self.threadpool.invoke(self.start_one_agent, node)
except:
self.threadpool.quit()
print sys.exc_info()[0], "DUCC may not be started correctly."
@@ -349,8 +340,8 @@ class StartDucc(DuccUtil):
pass # already started
else:
try:
- self.threadpool.invoke(self.start_component, ducc, com, or_parms, single_user)
- #self.start_component(ducc, com, or_parms, single_user)
+ self.threadpool.invoke(self.start_component, ducc, com, or_parms)
+ #self.start_component(ducc, com, or_parms)
except:
self.threadpool.quit()
print sys.exc_info()[0], "DUCC may not be started correctly."