You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2019/02/07 21:33:32 UTC
svn commit: r1853161 - in /uima/uima-ducc/trunk/src/main/admin: autostart.py
db_autostart_insert.py ducc_util.py
Author: degenaro
Date: Thu Feb 7 21:33:32 2019
New Revision: 1853161
URL: http://svn.apache.org/viewvc?rev=1853161&view=rev
Log:
UIMA-5742 Reliable DUCC
> autostart.py insert host+agent start into DB of never before seen node (one not already in DB)
> autostart.py don't use ssh for utility local host operations (e.g. query/start agent)
> ducc_util.py don't create database directories when automanage == false
Added:
uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py (with props)
Modified:
uima/uima-ducc/trunk/src/main/admin/autostart.py
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Modified: uima/uima-ducc/trunk/src/main/admin/autostart.py
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/autostart.py?rev=1853161&r1=1853160&r2=1853161&view=diff
==============================================================================
--- uima/uima-ducc/trunk/src/main/admin/autostart.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/autostart.py Thu Feb 7 21:33:32 2019
@@ -114,6 +114,10 @@ class AutoStart(DuccUtil):
'ws':'ws',
}
+ def __init__(self):
+ DuccUtil.__init__(self)
+ self.ssh_enabled = False
+
# return file name
def _fn(self):
fpath = __file__.split('/')
@@ -144,7 +148,6 @@ class AutoStart(DuccUtil):
LOGLEVEL = os.environ.get('LOGLEVEL','info')
self.logger = Logger(LOGFILE,LOGLEVEL)
-
# check if host names with domain match
def is_host_match_with_domain(self,h1,h2):
retVal = False
@@ -195,9 +198,10 @@ class AutoStart(DuccUtil):
pass
return retVal
- # get daemons started (from DB)
- def get_daemons_started(self):
- daemons = []
+ # get daemons started/all in DB for host
+ def get_daemons_host(self):
+ db = []
+ started = []
jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI'
option = '--query'
cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, jclass, option]
@@ -207,8 +211,9 @@ class AutoStart(DuccUtil):
for line in lines:
host, daemon, state = self.parse_line(line)
if(self.is_host_match(self.LOCAL_HOST, host)):
+ db.append(daemon)
if(state == 'Start'):
- daemons.append(daemon)
+ started.append(daemon)
text = 'add'+' '+host+' '+daemon
self.logger.debug(self._mn(),text)
else:
@@ -217,16 +222,16 @@ class AutoStart(DuccUtil):
else:
text = 'skip'+' '+host+' '+daemon
self.logger.debug(self._mn(),text)
- text = 'daemons'+' '+str(daemons)
+ text = 'started'+' '+str(started)+' '+'db'+' '+str(db)
self.logger.debug(self._mn(),text)
- return daemons
+ return db, started
def normalize_component(self,component):
daemon = component[:2]
return daemon
# get daemons running (from system)
- def get_daemons_running(self):
+ def get_components_running(self):
daemons = []
result = self.find_ducc_process(self.LOCAL_HOST)
find_status = result[0]
@@ -271,19 +276,50 @@ class AutoStart(DuccUtil):
out, err = p.communicate()
text = str(out)
self.logger.info(self._mn(),text)
-
+
+ def insert(self):
+ python_script = os.path.join(self.DUCC_HOME,'admin','db_autostart_insert.py')
+ node = self.get_node_name()
+ component = 'ag'
+ cmd = [ python_script, '--host', node, '--name', component]
+ text = str(cmd)
+ self.logger.info(self._mn(),text)
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = p.communicate()
+ text = str(out)
+ self.logger.info(self._mn(),text)
+
# autostart: start head or agent daemons, as required
def main(self, argv):
NAME = 'autostart'
self.setup_logging(NAME)
self.get_args()
try:
- daemons_started = self.get_daemons_started()
- daemons_running = self.get_daemons_running()
+ daemons_db, daemons_started = self.get_daemons_host()
+ text = 'daemons_db '+str(len(daemons_db))
+ self.logger.debug(self._mn(),text)
+ text = 'daemons_started '+str(len(daemons_started))
+ self.logger.debug(self._mn(),text)
+ # if agent node is not in db, then insert it!
+ if(len(daemons_db) == 0):
+ if(self.is_head_node()):
+ pass
+ else:
+ self.insert()
+ daemons_db, daemons_started = self.get_daemons_host()
+ components_running = self.get_components_running()
+ text = 'components_running '+str(components_running)
+ self.logger.debug(self._mn(),text)
for daemon in daemons_started:
- if(not daemon in daemons_running):
+ component = self.normalize_component(daemon)
+ text = 'component '+str(component)
+ self.logger.debug(self._mn(),text)
+ if(component in components_running):
+ pass
+ else:
+ text = 'start daemon '+str(daemon)
+ self.logger.debug(self._mn(),text)
self.start(daemon)
-
except Exception,e:
lines = traceback.format_exc().splitlines()
for line in lines:
Added: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py?rev=1853161&view=auto
==============================================================================
--- uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py (added)
+++ uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py Thu Feb 7 21:33:32 2019
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+import sys
+
+version_min = [2, 7]
+version_info = sys.version_info
+version_error = False
+if(version_info[0] < version_min[0]):
+ version_error = True
+elif(version_info[0] == version_min[0]):
+ if(version_info[1] < version_min[1]):
+ version_error = True
+if(version_error):
+ print('Python minimum requirement is version '+str(version_min[0])+'.'+str(version_min[1]))
+ sys.exit(1)
+
+import argparse
+import os
+import subprocess
+
+from ducc_util import DuccUtil
+
+# command to insert into the autostart database table the specified host & daemon
+
+class AutostartInsert(DuccUtil):
+
+ valid_names = [ 'ag', 'br', 'or', 'pm', 'rm', 'sm', 'ws' ]
+ jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI'
+
+ description = 'Insert an entry into the autostart database table, if one does not already exist.'
+
+ def get_args(self):
+ parser = argparse.ArgumentParser(description=self.description)
+ parser.add_argument('--host', action='store', required=True, help='the DUCC daemon host')
+ parser.add_argument('--name', action='store', required=True, choices=self.valid_names, help='the DUCC daemon name')
+ self.args = parser.parse_args()
+
+ def find(self):
+ retVal = False
+ option = '--query'
+ cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, self.jclass, option]
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = p.communicate()
+ lines = out.split('\n')
+ for line in lines:
+ tokens = line.split('.')
+ if(len(tokens) == 2):
+ host = tokens[0]
+ name = tokens[1].split('=')[0]
+ if(host == self.args.host):
+ if(name == self.args.name):
+ retVal = True
+ return retVal
+
+ def insert(self):
+ option = '--start'
+ cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, self.jclass, option, self.args.host, self.args.name]
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = p.communicate()
+
+ def main(self, argv):
+ self.get_args()
+ if(self.find()):
+ print 'already exists'
+ else:
+ self.insert()
+ if(self.find()):
+ print 'inserted'
+ else:
+ print 'insert failed'
+
+if __name__ == "__main__":
+
+ instance = AutostartInsert()
+ instance.main(sys.argv[1:])
Propchange: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py
------------------------------------------------------------------------------
svn:executable = *
Propchange: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1853161&r1=1853160&r2=1853161&view=diff
==============================================================================
--- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Thu Feb 7 21:33:32 2019
@@ -1451,10 +1451,17 @@ class DuccUtil(DuccBase):
if ( dbhost == None ):
dbhost = 'localhost'
- dir_db_state = self.DUCC_HOME + '/state/database/'+dbhost
- self.makedirs(dir_db_state)
- dir_db_logs = self.DUCC_HOME + '/logs/database/'+dbhost
- self.makedirs(dir_db_logs)
+
+ manage_database = self.ducc_properties.get('ducc.database.automanage')
+ self.automanage_database = False
+ if (manage_database in ('t', 'true', 'T', 'True')) :
+ self.automanage_database = True
+
+ if(manage_database):
+ dir_db_state = self.DUCC_HOME + '/state/database/'+dbhost
+ self.makedirs(dir_db_state)
+ dir_db_logs = self.DUCC_HOME + '/logs/database/'+dbhost
+ self.makedirs(dir_db_logs)
self.db_pidfile = dir_db_state+ '/cassandra.pid'
self.db_logfile = dir_db_logs + '/cassandra.console'
@@ -1472,11 +1479,6 @@ class DuccUtil(DuccBase):
if (manage_broker in ('t', 'true', 'T', 'True')) :
self.automanage_broker = True
- manage_database = self.ducc_properties.get('ducc.database.automanage')
- self.automanage_database = False
- if (manage_database in ('t', 'true', 'T', 'True')) :
- self.automanage_database = True
-
py_version = platform.python_version().split('.')
if ( int(py_version[0]) > 2 ):
print "Warning, only Python Version 2 is supported."