You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/09/15 16:31:25 UTC
svn commit: r1703203 [1/5] - in /uima/sandbox/uima-ducc/trunk: ./
src/main/admin/ src/main/assembly/ src/main/resources/
src/main/resources/private/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/
uima-ducc-common/src/main/java/org/apache/u...
Author: challngr
Date: Tue Sep 15 14:31:24 2015
New Revision: 1703203
URL: http://svn.apache.org/r1703203
Log:
UIMA-4577 First delivery, DB project.
Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/database.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/db.py (with props)
uima/sandbox/uima-ducc/trunk/src/main/admin/db_console (with props)
uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/DbQuery.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Server.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/Shutdown.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/misc/SmLoader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/resources/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/java/
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/test/resources/
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
Modified:
uima/sandbox/uima-ducc/trunk/pom.xml
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesSet.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccProperties.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/utilities/Checkpointable.java
uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccPerWorkItemStatistics.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkReservation.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java
Modified: uima/sandbox/uima-ducc/trunk/pom.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/pom.xml?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/pom.xml (original)
+++ uima/sandbox/uima-ducc/trunk/pom.xml Tue Sep 15 14:31:24 2015
@@ -121,6 +121,11 @@
</dependency>
<dependency>
<groupId>org.apache.uima</groupId>
+ <artifactId>uima-ducc-database</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
<artifactId>uima-ducc-user</artifactId>
<version>${project.version}</version>
</dependency>
@@ -175,6 +180,10 @@
</dependency>
<dependency>
<groupId>org.apache.uima</groupId>
+ <artifactId>uima-ducc-database</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uima</groupId>
<artifactId>uima-ducc-user</artifactId>
</dependency>
<dependency>
@@ -182,28 +191,13 @@
<artifactId>uima-ducc-container</artifactId>
</dependency>
</dependencies>
-
- <build>
-
-
- <!--resources>
- <resource>
- <directory>${baseidr}/src/main/resources</directory>
- <includes>
- <include>README</include>
- <include>RELEASE_NOTES.html</include>
- </includes>
- <filtering>true</filtering>
- <targetPath>${basedir}/target</targetPath>
- </resource>
- </resources-->
-
+ <build>
+ <resources />
<plugins>
-
<!-- This java doc config is for building the ones distributed with the
bin packaging, and also posted on our website. -->
<plugin>
@@ -466,9 +460,6 @@
</plugin -->
</plugins>
<!-- /pluginManagement -->
-
-
-
</build>
<!-- Include all ducc modules -->
@@ -485,6 +476,7 @@
<module>uima-ducc-duccdocs</module>
<module>uima-ducc-web</module>
<module>uima-ducc-examples</module>
+ <module>uima-ducc-database</module>
<module>uima-ducc-spawn</module>
<module>uima-ducc-user</module>
<!-- Note: uima-ducc-container has a test dependency on uima-ducc-user so must be built after it -->
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue Sep 15 14:31:24 2015
@@ -51,6 +51,16 @@ class CheckDucc(DuccUtil):
return
+ def verify_database(self):
+ if ( self.db_parms == self.db_disabled ):
+ return True
+
+ ret = self.db_alive()
+ if ( ret ):
+ print 'The database is running'
+ else:
+ print 'The database is not running'
+
def verify_activemq(self):
if ( self.is_amq_active() ):
print 'ActiveMQ is found listening at', self.broker_protocol + "://" + self.broker_host + ':' + self.broker_port
@@ -267,6 +277,8 @@ class CheckDucc(DuccUtil):
if ( not self.verify_activemq() ):
print 'ActiveMQ broker is not running on', self.broker_protocol + "://" + self.broker_host + ':' + self.broker_port
+ self.verify_database()
+
# init the PID file
self.pids = Properties()
self.pids.load_if_exists(self.pid_file)
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/database.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/database.py?rev=1703203&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/database.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/database.py Tue Sep 15 14:31:24 2015
@@ -0,0 +1,91 @@
+#!/usr/bin/python
+
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+# This is common code to deal wiht the orientdb configuration xml
+
+
+global domloaded
+try:
+ from xml.dom.minidom import parse
+ domloaded = True
+except ImportError:
+ domloaded = False
+
+
+def db_dom(dbconfig):
+ return parse(dbconfig)
+
+
+# <users>
+# <user resources="*" password="asdfasdf" name="root"/>
+# </users>
+
+def get_db_password(ducc_home, dbconfig):
+ dom = db_dom(ducc_home + '/' + dbconfig)
+
+ lst = dom.getElementsByTagName('users') # returns a Nodelist object
+ for u in lst:
+ childs = u.childNodes
+ for c in childs:
+ if (c.nodeName == 'user'):
+ atts = c.attributes
+ name = atts.getNamedItem('name').nodeValue
+ pw = atts.getNamedItem('password').nodeValue
+ if ( name == 'root' ):
+ return pw
+ return None
+
+def update_head(dbconfig, head):
+ # update the listeners with the correct head, return the dom
+ dom = db_dom(dbconfig)
+
+ lst = dom.getElementsByTagName('listener')
+ for l in lst:
+ atts = l.attributes
+ l.setAttribute('ip-address', head)
+ print 'Listener for protocol', atts.getNamedItem('protocol').value, 'set to', head
+ return dom
+
+def write_config(dbconfig, dom):
+ config = open(dbconfig, 'w')
+ config.write(dom.toxml())
+ config.close()
+
+def configure(dbconfig, head, pw):
+ print 'Configure', dbconfig, 'at', head, 'pw', pw
+
+ # parse the config, update the head node in the 'listeners' section
+ dom = update_head(dbconfig, head)
+
+ # set the root pw
+ lst = dom.getElementsByTagName('users') # returns a Nodelist object
+ for u in lst:
+ childs = u.childNodes
+ for c in childs:
+ if (c.nodeName == 'user'):
+ atts = c.attributes
+ name = atts.getNamedItem('name').nodeValue
+ if ( name == 'root' ):
+ c.setAttribute('password', pw)
+
+ # and rewrite the config
+ write_config(dbconfig, dom)
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/database.py
------------------------------------------------------------------------------
svn:executable = *
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/db.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db.py?rev=1703203&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db.py (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db.py Tue Sep 15 14:31:24 2015
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+import os
+import sys
+import getopt
+import httplib
+import base64
+import json
+import textwrap
+
+from ducc_util import DuccUtil
+from properties import Properties
+import database as db
+
+class DbUtil(DuccUtil):
+
+ def __init__(self):
+ DuccUtil.__init__(self)
+
+ def pprint(self, j):
+ print json.dumps(j, sort_keys=True, indent=4, separators=(',',':'))
+
+
+ #"clientId":"-",
+ #"commandDetail":"-",
+ #"commandInfo":"Listening",
+ #"connectedOn":"2015-08-05 06:21:30",
+ #"connectionId":"12",
+ #"db":"DuccHistory",
+ #"driver":"OrientDB Java v2.1-rc6 Protocol v30",
+ #"lastCommandDetail":"SELECT * FROM VRESERVATION WHERE ducc_dbid=944",
+ #"lastCommandInfo":"Execute remote command",
+ #"lastCommandOn":"2015-08-10 12:36:11",
+ #"lastExecutionTime":"3",
+ #"protocol":"binary",
+ #"remoteAddress":"/192.168.2.196:48994",
+ #"totalRequests":"39171",
+ #"totalWorkingTime":"196975",
+ #"user":"admin"
+
+ def format_connections(self, j):
+ keys = j[0].keys()
+ slen = {}
+ for k in keys:
+ maxv = len(k)
+ print 'initial', maxv
+ for c in j:
+ maxv = max(maxv, len(c[k]))
+ slen[k] = maxv
+
+ print slen
+ format = ''
+ for k in keys:
+ format = format + '%' + str(slen[k]) + 's | '
+
+
+ tmp = j[0]
+ self.pprint(j[0])
+ print format
+ print tmp
+ print len(tmp)
+ print format % (tuple(tmp.keys()))
+ print format % (tuple(tmp.values()))
+ return
+
+ i = 1
+ for c in j: # connectins in json
+ print 'Connection', i
+ i = i + 1
+ self.pprint(c)
+
+ maxl = 0
+ for k in c.keys():
+ maxl = max(maxl, len(k))
+ print 'max:', maxl
+
+
+ def show_connections(self):
+ print 'Show connections'
+
+ pw = base64.b64encode('root:asdfasdf')
+
+ try:
+ conn = httplib.HTTPConnection(self.dbrest)
+ conn.request('GET', '/server', None, {'Authorization':'Basic ' + pw})
+ except Exception, (e):
+ print " Checking connection: ", e
+ return False
+
+
+ resp = conn.getresponse()
+ #print 'response code', resp.status, resp.reason
+ data = resp.read()
+ #print 'Data:', data
+
+ if ( resp.status == 200 ):
+ #print data
+ conns = json.loads(data)
+ #print json.dumps(conns, sort_keys=True, indent=4, separators=(',',':'))
+
+ #self.pprint(conns['connections'])
+ #print conns['dbs']
+ #print conns['properties']
+ #print conns['storages']
+ print 'Connections:', len(conns['connections'])
+ self.format_connections(conns['connections'])
+
+
+ #
+ # Open an OrientDb console on the database defined for this DUCC instance
+ #
+ def open_console(self):
+
+ main = 'com.orientechnologies.orient.graph.console.OGremlinConsole'
+
+ jp = ''
+ for k in self.jvm_parms.keys():
+ v = self.jvm_parms[k]
+ if ( v == None ):
+ jp = jp + k + ' '
+ else:
+ jp = jp + k + '=' + v + ' '
+
+ cmd = ' '.join([self.java(), jp, '-cp', self.classpath, main, ])
+
+ here = os.getcwd()
+ os.chdir(self.db_rt)
+ pid = self.spawn(cmd)
+
+ def usage(self, *msg):
+ if ( msg[0] != None ):
+ print ' '.join(msg)
+
+ print 'Usage:'
+ print ' db.py [options]'
+ print ''
+ print 'Where options include:'
+ print ' -c show connection status'
+ print ''
+ print 'If no options are given a console to the database is opened.'
+ sys.exit(0)
+
+
+ def main(self, argv):
+
+ # The dom gets loaded by the import above - or not
+ if ( not db.domloaded ):
+ print "Unable to read database configuration; insure the installed Python supports xml.dom.minidom"
+ print "Note that Python must be at least version 2.6 but not 3.x. You are running version", sys.version_info
+ return
+
+ (self.jvm_parms, self.classpath, self.db_rt, self.dburl, self.dbrest, self.dbroot) = self.db_parms
+ try:
+ opts, args = getopt.getopt(argv, 'ch?', ['--connections', '--help'])
+ except:
+ self.usage('Bad arguments ' + ' '.join(argv))
+
+
+ if ( len(opts) == 0 ):
+ self.open_console()
+ return
+
+ for ( o, a ) in opts:
+ print 'o', o, 'a', a
+ if ( o in ['-c', '--connections']):
+ self.show_connections()
+ elif ( o in ['-h', '-?', '--help'] ):
+ self.usage()
+
+
+if __name__ == "__main__":
+ ducc = DbUtil()
+ ducc.main(sys.argv[1:])
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/db.py
------------------------------------------------------------------------------
svn:executable = *
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/db_console
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_console?rev=1703203&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_console (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_console Tue Sep 15 14:31:24 2015
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+
+from ducc_util import DuccUtil
+from properties import Properties
+import database as db
+
+class DbConsole(DuccUtil):
+
+ def __init__(self):
+ DuccUtil.__init__(self)
+
+ def main(self, rgv):
+ # The dom gets loaded by the import above - or not
+ if ( not db.domloaded ):
+ print "Unable to read database configuration; insure the installed Python supports xml.dom.minidom"
+ print "Note that Python must be at least version 2.6 but not 3.x. You are running version", sys.version_info
+ return
+
+
+ (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms()
+
+
+ main = 'com.orientechnologies.orient.graph.console.OGremlinConsole'
+
+ jp = ''
+ for k in jvm_parms.keys():
+ v = jvm_parms[k]
+ if ( v == None ):
+ jp = jp + k + ' '
+ else:
+ jp = jp + k + '=' + v + ' '
+
+ cmd = ' '.join([self.java(), jp, '-cp', classpath, main, ])
+
+ here = os.getcwd()
+ os.chdir(db_rt)
+ pid = self.spawn(cmd)
+
+
+if __name__ == "__main__":
+ console = DbConsole()
+ console.main(sys.argv[1:])
+
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/db_console
------------------------------------------------------------------------------
svn:executable = *
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue Sep 15 14:31:24 2015
@@ -27,9 +27,55 @@ import time
from ducc_util import DuccUtil
from local_hooks import verify_slave_node
+import database as db
+
+from properties import Properties
+from properties import Property
class Ducc(DuccUtil):
+
+ def run_db(self, background):
+ # The dom gets loaded by the import above - or not
+ if ( not db.domloaded ):
+ print "Unable to read database configuration; insure the installed Python supports xml.dom.minidom"
+ print "Note that Python must be at least version 2.6 but not 3.x. You are running version", sys.version_info
+ return
+
+ print '(re)Starting the database'
+
+ # first write this out so it's easy to find in java without having to parse xml
+ (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms
+ db_pw_file = Properties()
+ comment = []
+ comment.append('# Do not modify this file, it is auto-generated.')
+ comment.append('# This is extracted from the database configuration and written to this file on every boot of the db')
+ p = Property('db_password', dbroot, comment)
+ db_pw_file.put_property(p)
+ db_pw_filename = self.DUCC_HOME + '/resources.private/db_password'
+ db_pw_file.write(db_pw_filename)
+ os.chmod(db_pw_filename, 0700)
+
+ main = 'com.orientechnologies.orient.server.OServerMain'
+
+ jp = ''
+ for k in jvm_parms.keys():
+ v = jvm_parms[k]
+ if ( v == None ):
+ jp = jp + k + ' '
+ else:
+ jp = jp + k + '=' + v + ' '
+
+ cmd = ' '.join(['nohup', self.java(), jp, '-cp', classpath, main, '&'])
+ print cmd
+
+ here = os.getcwd()
+ os.chdir(db_rt)
+ pid = self.spawn(cmd)
+ os.chdir(here)
+
+ print 'OK'
+
def run_broker(self, component):
broker_port = self.ducc_properties.get('ducc.broker.port')
broker_jmx_port = self.ducc_properties.get('ducc.broker.jmx.port')
@@ -149,6 +195,7 @@ class Ducc(DuccUtil):
self.add_to_classpath(ducc_home + '/lib/http-client/*')
self.add_to_classpath(ducc_home + '/webserver/lib/*')
self.add_to_classpath(ducc_home + '/webserver/lib/jsp/*')
+ self.add_to_classpath(ducc_home + '/lib/orientdb/*')
if ( c == 'orchestrator' ):
if ( or_parms != None ):
@@ -157,6 +204,7 @@ class Ducc(DuccUtil):
jvm_opts.append(self.or_jvm_args)
self.add_to_classpath(ducc_home + '/lib/http-client/*')
self.add_to_classpath(ducc_home + '/webserver/lib/*')
+ self.add_to_classpath(ducc_home + '/lib/orientdb/*')
if ( c == 'pm' ):
if ( self.pm_jvm_args != None ):
@@ -168,6 +216,7 @@ class Ducc(DuccUtil):
self.add_to_classpath(ducc_home + '/apache-uima/apache-activemq/lib/optional/*')
self.add_to_classpath(ducc_home + '/lib/http-client/*')
self.add_to_classpath(ducc_home + '/webserver/lib/*')
+ self.add_to_classpath(ducc_home + '/lib/orientdb/*')
if (component != 'agent'):
service = 'org.apache.uima.ducc.common.main.DuccService'
@@ -290,10 +339,17 @@ class Ducc(DuccUtil):
if ( component == None ):
self.usage("Must specify component")
+ if ( component == 'db' ):
+ self.run_db(background)
+ return
+
if ( component == 'broker' ):
self.run_broker(background)
- else:
- self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate)
+ return
+
+ # fall-through, runs one of the ducc components proper
+ self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate)
+
return
def __call__(self, *args):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Tue Sep 15 14:31:24 2015
@@ -37,6 +37,8 @@ from ducc_base import which
from ducc import Ducc
+import database as db
+
class PostInstall():
def usage(self, msg):
@@ -78,6 +80,24 @@ class PostInstall():
print 'WARNING'
print ''
+ def configure_database(self):
+ self.ducc_default_properties = Properties()
+ self.ducc_default_properties.load(self.DUCC_HOME + '/resources/default.ducc.properties')
+ self.ducc_default_properties.put('ducc.head', self.ducc_head)
+
+ dburl = self.ducc_default_properties.get('ducc.database.url')
+ dbconfig = self.DUCC_HOME + '/' + self.ducc_default_properties.get('ducc.database.config.file')
+ print 'Configuring database at', dburl, 'from', dbconfig
+ print ''
+
+ dbpw = ''
+ while ( dbpw == '' ):
+ dbpw = raw_input('Enter password for database root:')
+
+ print ''
+ db.configure(dbconfig, self.ducc_head, dbpw)
+
+
def get_keystore_pw(self, prompt):
if ( self.keystore_pw == None ):
@@ -354,6 +374,8 @@ class PostInstall():
self.setup_ducc_head()
self.check_nodes()
+ self.configure_database()
+
# As of DUCC 2.0, always set here on installation
print 'ActiveMQ is automanaged on node ', self.localhost
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Sep 15 14:31:24 2015
@@ -28,11 +28,14 @@ import grp
import resource
import time
import platform
+import httplib
from threading import *
import traceback
import Queue
+import database as db
+
from stat import *
from local_hooks import find_other_processes
@@ -158,7 +161,190 @@ class DuccUtil(DuccBase):
CMD = ' '.join(CMD)
print 'Merging', base_props, 'with', site_props, 'into', run_props
os.system(CMD)
-
+
+ def db_read_parms(self):
+ #
+ # common setup for orientdb management
+ #
+
+ dbhost = self.ducc_properties.get('ducc.database.host')
+ if ( dbhost == self.db_disabled ):
+ self.db_parms = self.db_disabled
+ return self.db_parms
+
+ dburl = self.ducc_properties.get('ducc.database.url')
+ dbrest = self.ducc_properties.get('ducc.database.rest.url')
+ dbconfig = self.ducc_properties.get('ducc.database.config.file')
+ dbroot = db.get_db_password(DUCC_HOME, dbconfig)
+ if ( dbroot == None ):
+ print 'Cannot find database passord in', dbconfig
+ sys.exit(1)
+
+ rt = self.DUCC_HOME # (ducc runtime)
+ db_rt = rt + '/database' # ORIENTDB_HOME - the database "home" place
+
+ jvm_parms = {
+ '-Dfile.encoding' : 'UTF8',
+ '-Drhino.opt.level' : '9',
+ '-Dprofile.enabled' : 'true',
+ '-Djna.nosys' : 'true',
+ '-Djava.awt.headless' : 'true',
+ '-Dorientdb.config.file' : rt + '/' + dbconfig,
+ '-Dorientdb.www.path' : db_rt + '/www',
+ '-Djava.util.logging.config.file' : rt + '/resources/database.log.config',
+ }
+ classpath = '"' + rt + '/lib/orientdb/*'
+ classpath = classpath + ':' + rt + '/lib/jna/*' + '"'
+
+ os.environ['DUCC_HOME'] = self.DUCC_HOME
+ os.environ['ORIENTDB_HOME'] = db_rt
+
+ self.db_parms = (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot)
+
+ def db_alive(self):
+ if ( self.db_parms == self.db_disabled ):
+ return True
+
+ (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms
+
+ try:
+ conn = httplib.HTTPConnection(dbrest)
+ conn.request('GET', '/listDatabases')
+ except Exception, (e):
+ print " Checking connection: ", e
+ return False
+
+ resp = conn.getresponse()
+ #print 'response code', resp.status, resp.reason
+ data = resp.read()
+ #print 'Data:', data
+
+ if ( resp.status == 200 ):
+ # it will be simple json that Python will see as lists and maps so we can just eval it
+ data = eval(data)
+ dblist = data['databases']
+ if ( len(dblist) == 0 ):
+ print ' Connection succeeded, no databases found.'
+ pass
+ else:
+ print 'Found these databases:'
+ for d in data['databases']:
+ print ' ', d
+ return True
+
+ def db_start(self):
+
+ # bypass all of this for the initial delivery
+ if ( self.db_parms == self.db_disabled ):
+ return True
+
+ print 'Starting database'
+
+ self.sync_db_config() # insure db is running on head node, in case the head changed in ducc.properties
+ node = self.ducc_properties.get('ducc.head')
+
+ if ( node == 'local' ):
+ node = self.localhost
+
+ if ( self.db_alive() ):
+ print 'Database is already started.'
+ return True
+
+ max_attempts = 5
+ attempt = 0
+ while attempt < max_attempts:
+ lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'db', '--nodup', "'")
+ # we'll capture anything that the python shell spews because it may be useful, and then drop the
+ # pipe when we see a PID message
+ while True:
+ try:
+ line = lines.readline().strip()
+ except:
+ break
+ #print '[]', line
+
+ if ( not line ):
+ break
+ if ( line == '' ):
+ break
+ if ( line == 'OK' ):
+ print 'GOT OK from db start'
+ lines.close();
+
+ print 'waiting for database to start'
+ time.sleep(5)
+ if ( self.db_alive() ):
+ return True
+
+ attempt = attempt + 1
+ print 'Did not connect to database, retrying (', attempt, 'of', max_attempts, ')'
+
+ return False
+
+ def db_init(self):
+
+ # bypass all of this for the initial delivery
+ if ( self.db_parms == self.db_disabled ):
+ return True
+
+ print 'Initializing database.'
+ main = 'org.apache.uima.ducc.database.DbCreate'
+ (jvm_parms, classpath, db_rt, dburl, dbrest, dbroot) = self.db_parms
+
+ classpath = self.DUCC_HOME + '/lib/uima-ducc/*:' + classpath
+ dburl = self.ducc_properties.get('ducc.state.database.url')
+ cmd = ' '.join([self.java(), '-DDUCC_HOME=/home/challngr/ducc_runtime_db', '-cp', classpath, main, dburl])
+ print cmd
+ self.spawn(cmd)
+
+ def db_stop(self):
+ # bypass all of this for the initial delivery
+ if ( self.db_parms == self.db_disabled ):
+ return True
+
+
+ cfgfile = 'ducc.database.config.file'
+ dbconfig = self.ducc_properties.get(cfgfile)
+ if ( dbconfig == None ):
+ print 'Database is not configured:', cfgfile, 'is not found in ducc.properties'
+ return
+
+ rt = self.DUCC_HOME # (ducc runtime)
+ db_rt = rt + '/database' # ORIENTDB_HOME - the database "home" place
+
+ jvm_parms = {
+ '-Djava.awt.headless' : 'true',
+ '-Dorientdb.config.file' : rt + '/' + dbconfig,
+ }
+ classpath = '"' + rt + '/lib/orientdb/*' + '"'
+
+ main = 'com.orientechnologies.orient.server.OServerShutdownMain'
+
+ jp = ''
+ for k in jvm_parms.keys():
+ v = jvm_parms[k]
+ if ( v == None ):
+ jp = jp + k + ' '
+ else:
+ jp = jp + k + '=' + v + ' '
+
+
+ os.environ['ORIENTDB_HOME'] = db_rt
+ cmd = ' '.join([self.java(), jp, '-cp', classpath, main])
+ print cmd
+
+ here = os.getcwd()
+ os.chdir(db_rt)
+ self.spawn(cmd)
+ os.chdir(here)
+
+ def sync_db_config(self):
+ dbconfig = self.DUCC_HOME + '/' + self.ducc_properties.get('ducc.database.config.file')
+ head = self.ducc_properties.get('ducc.head')
+ print 'Sync database node with head node on', head
+ dom = db.update_head(dbconfig, head)
+ db.write_config(dbconfig, dom)
+
def find_netstat(self):
# don't you wish people would get together on where stuff lives?
if ( os.path.exists('/sbin/netstat') ):
@@ -765,12 +951,13 @@ class DuccUtil(DuccBase):
global use_threading
DuccBase.__init__(self, merge)
+ self.db_disabled = '--disabled--'
self.duccling = None
self.broker_url = 'tcp://localhost:61616'
self.broker_protocol = 'tcp'
self.broker_host = 'localhost'
self.broker_port = '61616'
- self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'broker']
+ self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'db', 'broker']
self.default_nodefiles = [self.DUCC_HOME + '/resources/ducc.nodes']
if ( self.localhost == self.ducc_properties.get("ducc.head")):
@@ -783,6 +970,9 @@ class DuccUtil(DuccBase):
self.os_pagesize = self.get_os_pagesize()
self.update_properties()
+ self.db_read_parms()
+
+
manage_broker = self.ducc_properties.get('ducc.broker.automanage')
self.automanage = False
if (manage_broker in ('t', 'true', 'T', 'True')) :
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue Sep 15 14:31:24 2015
@@ -311,6 +311,15 @@ class StartDucc(DuccUtil):
print sys.exc_info()[0], "DUCC may not be started correctly."
sys.exit(1)
+ try:
+ if ( self.db_start() ):
+ self.db_init()
+
+ except Exception (e):
+ # print e
+ print sys.exc_info()[0], "Can't start the database."
+ sys.exit(1)
+
if ( self.is_amq_active() ):
print 'ActiveMQ broker is found on configured host and port:', self.broker_host + ':' + self.broker_port
else:
@@ -336,7 +345,7 @@ class StartDucc(DuccUtil):
print 'Starting', or_parms
for com in components:
- if ( com == 'broker' ):
+ if ( com in ('broker', 'db') ):
pass # already started
else:
try:
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Tue Sep 15 14:31:24 2015
@@ -38,6 +38,10 @@ class StopDucc(DuccUtil):
self.stop_broker()
return
+ if ( component == 'db' ):
+ self.db_stop()
+ return
+
#
# If it's an unqualified management component, we need to get it's qualified name
#
@@ -78,6 +82,7 @@ class StopDucc(DuccUtil):
self.pids.delete(com)
self.pids.delete(component)
+
def quiesce_agents(self, components, nodes):
allnodes = []
for ( nf, nl ) in nodes.items():
@@ -134,6 +139,9 @@ class StopDucc(DuccUtil):
print ' ws - web server'
print ' agent - node agent'
print ''
+ print ' -w --wait'
+ print ' Time to wait for everything to come down, in seconds. Default is 60.'
+ print ''
print ' -k --kill'
print ' Stop the component forcibly and immediately using kill -9. Use this only if a'
print ' normal stop does not work (e.g. the process may be hung).'
@@ -167,9 +175,10 @@ class StopDucc(DuccUtil):
force = False
quiesce = False
all = False
+ wait_time = 60
try:
- opts, args = getopt.getopt(argv, 'ac:n:kn:qh?v', ['all', 'component=', 'components=', 'help', 'nodelist=', 'kill', 'quiesce', 'nothreading'])
+ opts, args = getopt.getopt(argv, 'ac:n:kn:w:qh?v', ['all', 'component=', 'components=', 'help', 'nodelist=', 'kill', 'quiesce', 'nothreading', 'wait'])
except:
self.invalid('Invalid arguments ' + ' '.join(argv))
@@ -187,6 +196,8 @@ class StopDucc(DuccUtil):
force = True
elif o in ( '-q', '--quiesce' ):
quiesce = True
+ elif o in ( '-w', '--wait' ):
+ wait_time = int(a)
elif o in ( '--nothreading' ):
self.disable_threading()
elif ( o == '-v' ) :
@@ -222,14 +233,15 @@ class StopDucc(DuccUtil):
self.clean_shutdown()
# Agents may wait up to 60 secs for processes to quiesce
- sleeptime = 60
- print "Waiting " + str(sleeptime) + " seconds to broadcast agent shutdown."
- time.sleep(sleeptime)
+ print "Waiting " + str(wait_time) + " seconds to broadcast agent shutdown."
+ time.sleep(wait_time)
if ( self.automanage ):
print "Stopping broker"
self.stop_broker()
+ self.db_stop()
+
if ( os.path.exists(self.pid_file) ):
os.remove(self.pid_file)
return
@@ -237,17 +249,27 @@ class StopDucc(DuccUtil):
if ( len(nodefiles) == 0 ):
nodefiles = self.default_nodefiles
+
self.pids = Properties()
- try:
- self.pids.load(self.pid_file)
- pass
- except PropertiesException, (inst):
- print inst.msg
- print ''
- print 'Run check_ducc to refresh the PIDs file, or check_ducc -k to search for and',
- print 'kill all DUCC processes.'
- print ''
- sys.exit(1)
+ sc = set(components)
+ sb = set(['broker', 'db'])
+ read_pids = True
+ if ( sc.issubset(sb) ):
+ read_pids = False
+
+
+ # The broker and db do not set the pid file
+ if ( read_pids ):
+ try:
+ self.pids.load(self.pid_file)
+ pass
+ except PropertiesException, (inst):
+ print inst.msg
+ print ''
+ print 'Run check_ducc to refresh the PIDs file, or check_ducc -k to search for and',
+ print 'kill all DUCC processes.'
+ print ''
+ sys.exit(1)
#
# if not 'all', we use nodefiles and component names
@@ -278,10 +300,11 @@ class StopDucc(DuccUtil):
for n in nl:
self.stop_agents(n, force)
- if ( len(self.pids) > 0 ):
- self.pids.write(self.pid_file)
- else:
- os.remove(self.pid_file)
+ if ( read_pids ):
+ if ( len(self.pids) > 0 ):
+ self.pids.write(self.pid_file)
+ else:
+ os.remove(self.pid_file)
return
Modified: uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml Tue Sep 15 14:31:24 2015
@@ -69,6 +69,7 @@ under the License.
<include>org.apache.uima:uima-ducc-orchestrator</include>
<include>org.apache.uima:uima-ducc-web</include>
<include>org.apache.uima:uima-ducc-container</include>
+ <include>org.apache.uima:uima-ducc-database</include>
</includes>
<outputFileNameMapping>${artifact.artifactId}-${project.version}.jar</outputFileNameMapping>
<outputDirectory>lib/uima-ducc</outputDirectory>
@@ -129,6 +130,80 @@ under the License.
<directoryMode>755</directoryMode>
</dependencySet>
+ <!-- orientdb -->
+ <dependencySet>
+ <useStrictFiltering>true</useStrictFiltering>
+ <includes>
+ <include>com.tinkerpop.gremlin:gremlin-groovy</include>
+ </includes>
+ <outputDirectory>lib/orientdb</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ </dependencySet>
+
+ <dependencySet>
+ <useStrictFiltering>true</useStrictFiltering>
+ <includes>
+ <include>com.tinkerpop.blueprints:blueprints-core</include>
+ </includes>
+ <outputDirectory>lib/orientdb</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ </dependencySet>
+
+ <dependencySet>
+ <useStrictFiltering>true</useStrictFiltering>
+ <includes>
+ <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include>
+ </includes>
+ <outputDirectory>lib/orientdb</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ </dependencySet>
+
+ <dependencySet>
+ <useStrictFiltering>true</useStrictFiltering>
+ <includes>
+ <include>com.orientechnologies:orientdb-graphdb</include>
+ <include>com.orientechnologies:orientdb-core</include>
+ <include>com.orientechnologies:orientdb-client</include>
+ <include>com.orientechnologies:orientdb-server</include>
+ <include>com.orientechnologies:orientdb-tools</include>
+ <include>com.orientechnologies:orientdb-enterprise</include>
+ </includes>
+ <outputDirectory>lib/orientdb</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ </dependencySet>
+
+ <dependencySet>
+ <useStrictFiltering>true</useStrictFiltering>
+ <includes>
+ <include>com.tinkerpop.gremlin:gremlin-groovy</include>
+ </includes>
+ <outputDirectory>lib/orientdb</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ </dependencySet>
+
+ <!-- dependencySet>
+ <useStrictFiltering>true</useStrictFiltering>
+ <includes>
+ <include>com.orientechnologies:orientdb-lucene</include>
+ </includes>
+ <outputDirectory>database/plugins</outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ </dependencySet -->
+
+ <!-- end orientdb -->
+
<!-- xmlbeans is in UIMA-AS 2.6.0 package -->
<dependencySet>
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Tue Sep 15 14:31:24 2015
@@ -502,4 +502,35 @@ ducc.process.thread.sleep.time=3000
ducc.flow-controller.specifier=org.apache.uima.ducc.FlowController
+# database things -----------------------------------------------------------------------------------
+# Directory where the databases and associated stuff is kept, relative to DUCC_HOME. This
+# is ORIENTDB_HOME
+ducc.database.location = database
+
+# DB defined config file, relative to DUCC_HOME. Must be in private because it has root pw in plaintext.
+ducc.database.config.file = resources.private/database.xml
+
+ducc.database.host = --disabled--
+
+# The binary url not HTTP/REST
+ducc.database.url = remote:${ducc.database.host}:2424
+# The rest interface
+ducc.database.rest.url = ${ducc.database.host}:2480
+
+# Name of the database with active state. Svc registry, etc.
+ducc.state.database.url = ${ducc.database.url}/DuccState
+
+# Name of the database with history
+ducc.history.database.url = ${ducc.database.url}/DuccHistory
+
+# This class implementes the persistence interface for the services registry
+#ducc.service.persistence.impl = org.apache.uima.ducc.database.StateServicesDb
+ducc.service.persistence.impl = org.apache.uima.ducc.common.persistence.services.StateServices
+
+# This class implements the persistence interface for Job, Reservation, Service, and AP instances, plus
+# the Orchestrator checkpoint.
+#ducc.job.history.impl = org.apache.uima.ducc.database.HistoryManagerDb
+ducc.job.history.impl = org.apache.uima.ducc.transport.event.common.history.HistoryPersistenceManager
+
+# end database things -------------------------------------------------------------------------------
Added: uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml?rev=1703203&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/private/database.xml Tue Sep 15 14:31:24 2015
@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<orient-server>
+ <handlers>
+ <handler class="com.orientechnologies.orient.graph.handler.OGraphServerHandler">
+ <parameters>
+ <parameter value="true" name="enabled"/>
+ <parameter value="50" name="graph.pool.max"/>
+ </parameters>
+ </handler>
+
+ <!-- This sets up the redundant distributed server setup -->
+ <!-- Configured out for now, jrc 2015-06-18 -->
+ <!--
+ <handler class="com.orientechnologies.orient.server.hazelcast.OHazelcastPlugin">
+ <parameters>
+ <parameter value="${distributed}" name="enabled"/>
+ <parameter value="${ORIENTDB_HOME}/config/default-distributed-db-config.json" name="configuration.db.default"/>
+ <parameter value="${ORIENTDB_HOME}/config/hazelcast.xml" name="configuration.hazelcast"/>
+ </parameters>
+ </handler>
+ -->
+
+ <!-- JMX Support. -->
+ <!-- Configured out for now. jrc 2-15-06-18 -->
+ <!--
+ <handler class="com.orientechnologies.orient.server.handler.OJMXPlugin">
+ <parameters>
+ <parameter value="true" name="enabled"/>
+ <parameter value="true" name="profilerManaged"/>
+ </parameters>
+ </handler>
+ -->
+
+ <handler class="com.orientechnologies.orient.server.handler.OAutomaticBackup">
+ <parameters>
+ <parameter value="true" name="enabled"/>
+ <parameter value="4h" name="delay"/>
+ <parameter value="backup" name="target.directory"/>
+ <parameter value="${DBNAME}-${DATE:yyyyMMddHHmmss}.zip" name="target.fileName"/>
+ <parameter value="9" name="compressionLevel"/>
+ <parameter value="1048576" name="bufferSize"/>
+ <parameter value="" name="db.include"/>
+ <parameter value="" name="db.exclude"/>
+ </parameters>
+ </handler>
+
+ <!-- Server side scripting, dangerous and unnecessary at this point. -->
+ <!-- Disabled by jrc 2015-06-28 -->
+ <!--
+ <handler class="com.orientechnologies.orient.server.handler.OServerSideScriptInterpreter">
+ <parameters>
+ <parameter value="true" name="enabled"/>
+ <parameter value="SQL" name="allowedLanguages"/>
+ </parameters>
+ </handler>
+ -->
+ <handler class="com.orientechnologies.orient.server.token.OrientTokenHandler">
+ <parameters>
+ <parameter value="false" name="enabled"/>
+ <parameter value="" name="oAuth2Key"/>
+ <parameter value="60" name="sessionLength"/>
+ <parameter value="HmacSHA256" name="encryptionAlgorithm"/>
+ </parameters>
+ </handler>
+ </handlers>
+ <network>
+ <sockets>
+ <socket implementation="com.orientechnologies.orient.server.network.OServerSSLSocketFactory" name="ssl">
+ <parameters>
+ <parameter value="false" name="network.ssl.clientAuth"/>
+ <parameter value="config/cert/orientdb.ks" name="network.ssl.keyStore"/>
+ <parameter value="password" name="network.ssl.keyStorePassword"/>
+ <parameter value="config/cert/orientdb.ks" name="network.ssl.trustStore"/>
+ <parameter value="password" name="network.ssl.trustStorePassword"/>
+ </parameters>
+ </socket>
+ <socket implementation="com.orientechnologies.orient.server.network.OServerSSLSocketFactory" name="https">
+ <parameters>
+ <parameter value="false" name="network.ssl.clientAuth"/>
+ <parameter value="config/cert/orientdb.ks" name="network.ssl.keyStore"/>
+ <parameter value="password" name="network.ssl.keyStorePassword"/>
+ <parameter value="config/cert/orientdb.ks" name="network.ssl.trustStore"/>
+ <parameter value="password" name="network.ssl.trustStorePassword"/>
+ </parameters>
+ </socket>
+ </sockets>
+ <protocols>
+ <protocol implementation="com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary" name="binary"/>
+ <protocol implementation="com.orientechnologies.orient.server.network.protocol.http.ONetworkProtocolHttpDb" name="http"/>
+ </protocols>
+ <listeners>
+ <!-- Original has a set of listeners, we want to restrict to just one in DUCC -->
+ <!--
+ <listener protocol="binary" socket="default" port-range="2424-2430" ip-address="0.0.0.0"/>
+ <listener protocol="http" socket="default" port-range="2480-2490" ip-address="0.0.0.0">
+ -->
+ <listener protocol="binary" socket="default" port-range="2424" ip-address="replaceme" />
+ <listener protocol="http" socket="default" port-range="2480" ip-address="replaceme" >
+ <commands>
+ <command implementation="com.orientechnologies.orient.server.network.protocol.http.command.get.OServerCommandGetStaticContent" pattern="GET|www GET|studio/ GET| GET|*.htm GET|*.html GET|*.xml GET|*.jpeg GET|*.jpg GET|*.png GET|*.gif GET|*.js GET|*.css GET|*.swf GET|*.ico GET|*.txt GET|*.otf GET|*.pjs GET|*.svg GET|*.json GET|*.woff GET|*.ttf GET|*.svgz" stateful="false">
+ <parameters>
+ <entry value="Cache-Control: no-cache, no-store, max-age=0, must-revalidate\r\nPragma: no-cache" name="http.cache:*.htm *.html"/>
+ <entry value="Cache-Control: max-age=120" name="http.cache:default"/>
+ </parameters>
+ </command>
+ <command implementation="com.orientechnologies.orient.graph.server.command.OServerCommandGetGephi" pattern="GET|gephi/*" stateful="false"/>
+ </commands>
+ <parameters>
+ <parameter value="utf-8" name="network.http.charset"/>
+ </parameters>
+ </listener>
+ </listeners>
+ </network>
+ <storages/>
+ <users>
+ <user resources="*" password="replace" name="root"/>
+ <user resources="connect,server.listDatabases" password="guest" name="guest"/>
+ </users>
+ <properties>
+ <entry value="1" name="db.pool.min"/>
+ <entry value="50" name="db.pool.max"/>
+ <entry value="false" name="profiler.enabled"/> <!-- was true, made false, jrc 2015-06-18 -->
+ <entry value="info" name="log.console.level"/>
+ <entry value="info" name="log.file.level"/>
+ </properties>
+</orient-server>
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/IDuccEnv.java Tue Sep 15 14:31:24 2015
@@ -41,7 +41,8 @@ public interface IDuccEnv {
public static final String DUCC_HISTORY_RESERVATIONS_DIR = DUCC_HOME_DIR+"history"+File.separator+"reservations"+File.separator;
public static final String DUCC_HISTORY_SERVICES_DIR = DUCC_HOME_DIR+"history"+File.separator+"services"+File.separator;
- public static final String DUCC_STATE_SERVICES_DIR = DUCC_HOME_DIR+"state"+File.separator+"services"+File.separator;
+ public static final String DUCC_STATE_SVCREG_DIR = DUCC_HOME_DIR+"state"+File.separator+"services"+File.separator;
+ public static final String DUCC_HISTORY_SVCREG_DIR = DUCC_HOME_DIR+"history"+File.separator+"services-registry"+File.separator;
public static final String DUCC_LOGS_DIR = DUCC_HOME_DIR+"logs"+File.separator;
public static final String DUCC_LOGS_WEBSERVER_DIR = DUCC_HOME_DIR+"logs"+File.separator+"webserver"+File.separator;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java Tue Sep 15 14:31:24 2015
@@ -18,51 +18,128 @@
*/
package org.apache.uima.ducc.common.persistence.services;
-import java.io.IOException;
-import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
import org.apache.uima.ducc.common.IDuccEnv;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
public interface IStateServices {
- public static String directory_state_services = IDuccEnv.DUCC_STATE_SERVICES_DIR;
+ public static String svc_reg_dir = IDuccEnv.DUCC_STATE_SVCREG_DIR;
+ public static String svc_hist_dir = IDuccEnv.DUCC_HISTORY_SVCREG_DIR;
public static final String svc = "svc";
public static final String meta = "meta";
+
+ public enum SvcProps {
+ endpoint { public String pname() { return "endpoint"; } },
+ instances { public String pname() { return "instances"; } },
+ autostart { public String pname() { return "autostart"; } },
+ reference { public String pname() { return "reference"; } },
+ enabled { public String pname() { return "enabled"; } },
+
+ disable_reason { public String pname() { return "disable_reason"; } },
+
+ implementors { public String pname() { return "implementors"; } },
+ numeric_id { public String pname() { return "numeric_id"; } },
+ ping_active { public String pname() { return "ping_active"; } },
+ ping_only { public String pname() { return "ping_only"; } },
+
+ service_alive { public String pname() { return "service_alive"; } },
+ service_class { public String pname() { return "service_class"; } },
+ service_dependency { public String pname() { return "service_dependency"; } },
+ service_healthy { public String pname() { return "service_healthy"; } },
+ service_state { public String pname() { return "service_state"; } },
+ last_use { public String pname() { return "last_use"; } },
+ last_use_readable { public String pname() { return "last_use_readable"; } },
+ service_statistics { public String pname() { return "service_statistics"; } },
+ service_type { public String pname() { return "service_type"; } },
+ submit_error { public String pname() { return "submit_error"; } },
+ user { public String pname() { return "user"; } },
+
+ references { public String pname() { return "references"; } },
+ stopped { public String pname() { return "stopped"; } },
+
+ scheduling_class { public String pname() { return "scheduling_class"; } },
+ process_memory_size { public String pname() { return "process_memory_size"; } },
+ description { public String pname() { return "description"; } },
+ log_directory { public String pname() { return "log_directory"; } },
+ process_executable { public String pname() { return "process_executable"; } },
+
+ last_ping { public String pname() { return "last_ping"; } },
+ last_ping_readable { public String pname() { return "last_ping_readable"; } },
+ last_runnable { public String pname() { return "last_runnable"; } },
+ last_runnable_readable { public String pname() { return "last_runnable_readable"; } },
+ work_instances { public String pname() { return "work_instances"; } },
+ registration_date { public String pname() { return "registration_date"; } },
+
+ instance_init_failures_limit { public String pname() { return "instance_init_failures_limit"; } },
+
+ UIMA_AS { public String pname() { return "UIMA-AS"; } },
+ CUSTOM { public String pname() { return "CUSTOM"; } },
+
+ ;
+ public abstract String pname();
+
+ };
+
+ //public static final String endpoint { public String pname() { return "endpoint"; } },
+ //public static final String instances = "instances";
+ //public static final String autostart = "autostart";
+
+ //public static final String reference = "reference";
+ //public static final String enabled = "enabled";
+ //public static final String disable_reason = "disable-reason";
+ //public static final String implementors = "implementors";
+ //public static final String numeric_id = "numeric_id";
+ //public static final String ping_active = "ping-active";
+ //public static final String ping_only = "ping-only";
+ //public static final String service_alive = "service-alive";
+ //public static final String service_class = "service-class";
+ //public static final String service_dependency = "service_dependency";
+ //public static final String service_healthy = "service-healthy";
+ //public static final String service_state = "service-state";
+ //public static final String last_use = "last-use";
+ //public static final String service_statistics = "service-statistics";
+ //public static final String service_type = "service-type";
+ //public static final String submit_error = "submit-error";
+ //public static final String user = "user";
- public static final String endpoint = "endpoint";
- public static final String instances = "instances";
- public static final String autostart = "autostart";
- public static final String reference = "reference";
- public static final String enabled = "enabled";
- public static final String disable_reason = "disable-reason";
- public static final String implementors = "implementors";
- public static final String numeric_id = "numeric_id";
- public static final String ping_active = "ping-active";
- public static final String ping_only = "ping-only";
- public static final String service_alive = "service-alive";
- public static final String service_class = "service-class";
- public static final String service_dependency = "service_dependency";
- public static final String service_healthy = "service-healthy";
- public static final String service_state = "service-state";
- public static final String last_use = "last-use";
- public static final String service_statistics = "service-statistics";
- public static final String service_type = "service-type";
- public static final String submit_error = "submit-error";
- public static final String user = "user";
-
- public static final String scheduling_class = "scheduling_class";
- public static final String process_memory_size = "process_memory_size";
- public static final String description = "description";
- public static final String log_directory = "log_directory";
- public static final String process_executable = "process_executable";
+ //public static final String scheduling_class = "scheduling_class";
+ //public static final String process_memory_size = "process_memory_size";
+ //public static final String description = "description";
+ //public static final String log_directory = "log_directory";
+ //public static final String process_executable = "process_executable";
- public static final String service_type_UIMA_AS = "UIMA-AS";
- public static final String service_type_CUSTOM = "CUSTOM";
+ //public static final String service_type_UIMA_AS = "UIMA-AS";
+ //public static final String service_type_CUSTOM = "CUSTOM";
- public ArrayList<String> getSvcList();
- public ArrayList<String> getMetaList();
+ //
+ // IMPORTANT IMPORTANT IMPORTANT
+ // As of now, 2015/08/17, there is no support for service registration history. There never has been in
+ // this class and it's derivatives. Until there is a use-case and demand it's not clear what the interfaces
+ // would be like.
+ //
+ // We may have to prune, or look at specific history files, but that can be done with direct database
+ // operations. If we need more, well add it later.
+ //
+ // The only nod to history we make here is the ability to move a registration and its meta from 'live' to
+ // 'history' state.
+ // IMPORTANT IMPORTANT IMPORTANT
+
+ public List<Long> getSvcList() throws Exception; // list of registered services
+ public List<Long> getMetaList() throws Exception; // not used ?
- public StateServicesDirectory getStateServicesDirectory() throws IOException;
+ public StateServicesDirectory getStateServicesDirectory() throws Exception; // all the registy in one blow
+
+ public boolean storeProperties (DuccId serviceId, Properties svc, Properties meta) throws Exception; // save svc and meta in a transaction
+ public boolean updateJobProperties (DuccId serviceId, Properties props) throws Exception; // update just job props
+ public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception; // update just metaprops
+ public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception;
+
+ public void shutdown() throws Exception;
+ public boolean init(DuccLogger logger) throws Exception;
}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java?rev=1703203&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java Tue Sep 15 14:31:24 2015
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.persistence.services;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Properties;
+
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+
+
+public class NullStateServices implements IStateServices
+{
+
+ NullStateServices() {
+ }
+
+ public boolean init(DuccLogger logger) throws Exception { return true; }
+ public ArrayList<Long> getSvcList()
+ throws Exception
+ {
+ return new ArrayList<Long>();
+ }
+
+
+ public ArrayList<Long> getMetaList()
+ throws Exception
+ {
+ return new ArrayList<Long>();
+ }
+
+
+ public StateServicesDirectory getStateServicesDirectory()
+ throws IOException
+ {
+ return new StateServicesDirectory();
+ }
+
+ public boolean storeProperties(DuccId serviceId, Properties svc, Properties meta) throws Exception {return false;}
+ public boolean updateProperties(DuccId serviceId, String type, Properties props) throws Exception {return false;}
+ public void deleteProperties(DuccId serviceId) throws Exception {}
+ public void shutdown() throws Exception {}
+ public void moveToHHistory() throws Exception {}
+ public boolean updateJobProperties(DuccId serviceId, Properties props) throws Exception {return false;}
+ public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception {return false;}
+ public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception {}
+
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java Tue Sep 15 14:31:24 2015
@@ -20,70 +20,87 @@ package org.apache.uima.ducc.common.pers
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Properties;
-import org.apache.uima.ducc.common.IDuccEnv;
-import org.apache.uima.ducc.common.main.DuccService;
import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccProperties;
import org.apache.uima.ducc.common.utils.IOHelper;
import org.apache.uima.ducc.common.utils.id.DuccId;
public class StateServices implements IStateServices {
- private static DuccLogger logger = DuccService.getDuccLogger(StateServices.class.getName());
- private static DuccId jobid = null;
-
- private String directory_state_services = IDuccEnv.DUCC_STATE_SERVICES_DIR;
+ private DuccLogger logger = null;
StateServices() {
- mkdirs();
}
-
+
+ public boolean init(DuccLogger logger)
+ {
+ this.logger = logger;
+ mkdirs();
+ return true;
+ }
+
private void mkdirs() {
- IOHelper.mkdirs(directory_state_services);
+ IOHelper.mkdirs(svc_reg_dir);
+ IOHelper.mkdirs(svc_hist_dir);
}
-
- private ArrayList<String> getList(String type) {
+
+ private String mkfilename(long id, String type)
+ {
+ return svc_reg_dir + Long.toString(id) + "." + type;
+ }
+
+ private String mkfilename(DuccId id, String type)
+ {
+ return mkfilename(id.getFriendly(), type);
+ }
+
+ private ArrayList<Long> getList(String type) {
String location = "getList";
- ArrayList<String> retVal = new ArrayList<String>();
+ ArrayList<Long> retVal = new ArrayList<Long>();
try {
- logger.debug(location, jobid, directory_state_services);
- File folder = new File(directory_state_services);
+ logger.debug(location, null, svc_reg_dir);
+ File folder = new File(svc_reg_dir);
File[] listOfFiles = folder.listFiles();
if(listOfFiles != null) {
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile()) {
String name = listOfFiles[i].getName();
if(name.endsWith("."+type)) {
- retVal.add(name);
+ int ndx = name.lastIndexOf(".");
+ name = name.substring(0, ndx);
+ retVal.add(Long.parseLong(name));
}
}
}
}
}
catch(Exception e) {
- logger.error(location, jobid, e);
+ logger.error(location, null, e);
}
return retVal;
}
- public ArrayList<String> getSvcList() {
+ public ArrayList<Long> getSvcList() {
return getList(IStateServices.svc);
}
- public ArrayList<String> getMetaList() {
+ public ArrayList<Long> getMetaList() {
return getList(IStateServices.meta);
}
- private Properties getProperties(String name) {
+ private DuccProperties getProperties(String name) {
String location = "getProperties";
- Properties properties = new Properties();
+ DuccProperties properties = new DuccProperties();
try {
FileInputStream fis = new FileInputStream(name);
try {
@@ -94,43 +111,201 @@ public class StateServices implements IS
}
}
catch(Exception e) {
- logger.error(location, jobid, e);
+ logger.error(location, null, e);
}
return properties;
}
- public StateServicesDirectory getStateServicesDirectory() throws IOException {
+ public StateServicesDirectory getStateServicesDirectory()
+ throws Exception
+ {
String location = "getStateServicesDirectory";
StateServicesDirectory ssd = null;
try {
ssd = new StateServicesDirectory();
- ArrayList<String> svcList = getSvcList();
- logger.trace(location, jobid, svcList.size());
- for(String entry : svcList) {
+ ArrayList<Long> svcList = getSvcList();
+ logger.trace(location, null, svcList.size());
+ for(Long entry : svcList) {
try {
- StateServicesSet sss = new StateServicesSet();
- String num = entry.split("[.]")[0];
- Integer i = new Integer(num);
- String base = directory_state_services+num;
- logger.trace(location, jobid, base);
- String fnSvc = base+"."+svc;
- String fnMeta = base+"."+meta;
- Properties propertiesSvc = getProperties(fnSvc);
- sss.put(svc, propertiesSvc);
- Properties propertiesMeta = getProperties(fnMeta);
- sss.put(meta, propertiesMeta);
- ssd.put(i, sss);
+ StateServicesSet sss = new StateServicesSet();
+ String fnSvc = mkfilename(entry, svc);
+ String fnMeta = mkfilename(entry, meta);
+ DuccProperties propertiesSvc = getProperties(fnSvc);
+ sss.put(svc, propertiesSvc);
+ DuccProperties propertiesMeta = getProperties(fnMeta);
+ sss.put(meta, propertiesMeta);
+ ssd.put(entry, sss);
}
catch(Exception e) {
- logger.error(location, jobid, e);
+ logger.error(location, null, e);
}
}
}
catch(Exception e) {
- logger.error(location, jobid, e);
+ logger.error(location, null, e);
}
return ssd;
}
-
+
+ // Try to write properties file, using a temp file as backup in case it fails.
+ private boolean writeProperties(DuccId id, Properties props, File pfile, File pfile_tmp, String type)
+ {
+
+ String methodName = "saveProperties";
+ FileOutputStream fos = null;
+
+ long original_size = pfile.length();
+ try {
+ if ( (!pfile.exists()) || pfile.renameTo(pfile_tmp) ) {
+ fos = new FileOutputStream(pfile);
+ props.store(fos, type + " Descriptor");
+ } else {
+ logger.error(methodName, id, "Cannot save", type, "properties, rename of", pfile, "to", pfile_tmp, "fails.");
+ if ( (!pfile.exists()) && pfile_tmp.exists() ) {
+ if ( !pfile_tmp.renameTo(pfile) ) {
+ logger.error(methodName, id, "Cannot restore", pfile_tmp, "to", pfile, "after failed update.");
+ }
+ }
+ }
+ } catch (FileNotFoundException e) {
+ logger.error(methodName, id, "Cannot save " + type + " properties, file does not exist.");
+ } catch (IOException e) {
+ logger.error(methodName, id, "I/O Error saving " + type + " service properties: " + e.toString());
+ } catch (Throwable t) {
+ logger.error(methodName, id, "Unexpected Error saving " + type + " service properties: " + t.toString());
+ } finally {
+ try {
+ if ( fos != null ) fos.close();
+ long updated_size = pfile.length();
+ long tmp_size = pfile_tmp.length();
+
+ // updated size must be > 0 and tmp_size must match original size
+ if ( (updated_size > 0) && (original_size == tmp_size) ) {
+ pfile_tmp.delete();
+ } else {
+ logger.error(methodName, id, "Update of", pfile.toString(), "failed. Original size:", original_size, "updated size", updated_size, "temp file size", tmp_size);
+ logger.error(methodName, id, "The updated size must be > 0 and the temp size must match the original size for sucess.");
+ logger.error(methodName, id, "Attempting to restore", pfile.toString(), "from", pfile_tmp.toString());
+ if ( !pfile.exists() && pfile_tmp.exists() ) {
+ pfile_tmp.renameTo(pfile);
+ }
+ return false;
+ }
+ } catch (Throwable t) {
+ logger.error(methodName, id, "Cannot close", type, "properties:", t);
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ // Try, and retry, to save a single props file. Returns true if ok, false otherwise.
+ private boolean saveProperties(DuccId id, Properties props, File pfile, File pfile_tmp, String type)
+ {
+ int max = 5;
+ for ( int i = 0; i < max; i++ ) {
+ if (writeProperties(id, props, pfile, pfile_tmp, type)) return true;
+ }
+ return false;
+ }
+
+
+ // Attempt to transactionally save the two service props. If it returns null
+ // the caller can assume it worked. Otherwise and error string indicating the cause
+ // of failure is returned. In case of failure, we try to insure no properties
+ // file linger.
+ public boolean storeProperties (DuccId id, Properties svcprops, Properties metaprops)
+ {
+ // save svc and meta in a transaction
+
+ File svcfile = new File(mkfilename(id, svc));
+ File metafile = new File(mkfilename(id, meta));
+ boolean ok = true;
+
+ File tmpfile = new File(svcfile.toString() + ".tmp");
+ if ( saveProperties(id, svcprops, svcfile, tmpfile, svc) ) {
+ tmpfile = new File(metafile.toString() + ".tmp");
+ ok = saveProperties(id, metaprops, metafile, tmpfile, meta);
+ }
+
+ if ( !ok ) {
+ metafile.delete();
+ svcfile.delete();
+ }
+ return ok;
+ }
+
+ private boolean updateProperties(DuccId serviceId, Properties props, String type)
+ {
+ File f = new File(mkfilename(serviceId, type));
+ File tmpf = new File(f.toString() + ".tmp");
+ return saveProperties(serviceId, props, f, tmpf, type);
+ }
+
+ public boolean updateJobProperties(DuccId serviceId, Properties props)
+ {
+ return updateProperties(serviceId, props, svc);
+ }
+
+ public boolean updateMetaProperties(DuccId serviceId, Properties props)
+ {
+ return updateProperties(serviceId, props, meta);
+ }
+
+ public void deleteProperties(long serviceId)
+ {
+ String svcfile = mkfilename(serviceId, svc);
+ String metafile = mkfilename(serviceId, meta);
+
+ File pf = new File(svcfile);
+ pf.delete();
+
+ File mf = new File(metafile);
+ mf.delete();
+ }
+
+ public void deleteProperties(DuccId id)
+ {
+ deleteProperties(id.getFriendly());
+ }
+
+
+ public void moveToHistory(DuccId id, Properties svc, Properties meta)
+ throws Exception
+ {
+ String methodName = "moveToHistory";
+
+ // Save a copy in history, and then delete the original
+ File mfh = new File(svc_hist_dir + id + ".meta");
+ try {
+ FileOutputStream fos = new FileOutputStream(mfh);
+ meta.store(fos, "Archived meta descriptor");
+ fos.close();
+ } catch (Exception e) {
+ logger.warn(methodName, null, id + ": Unable to save history to \"" + mfh.toString(), ": ", e.toString() + "\"");
+ }
+
+ String meta_filename = svc_reg_dir + id + ".meta";
+ File mf = new File(meta_filename);
+ mf.delete();
+
+
+ File pfh = new File(svc_hist_dir + id + ".svc");
+
+ try {
+ FileOutputStream fos = new FileOutputStream(pfh);
+ svc.store(fos, "Archived svc properties.");
+ fos.close();
+ } catch (Exception e) {
+ logger.warn(methodName, null, id + ":Unable to save history to \"" + pfh.toString(), ": ", e.toString() + "\"");
+ }
+
+ String props_filename = svc_reg_dir + id + ".svc";
+ File pf = new File(props_filename);
+ pf.delete();
+ }
+
+ public void shutdown() {}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java Tue Sep 15 14:31:24 2015
@@ -23,17 +23,17 @@ import java.util.TreeMap;
public class StateServicesDirectory {
- private TreeMap<Integer,StateServicesSet> map = new TreeMap<Integer,StateServicesSet>();
+ private TreeMap<Long,StateServicesSet> map = new TreeMap<Long,StateServicesSet>();
- public void put(Integer key, StateServicesSet value) {
+ public void put(Long key, StateServicesSet value) {
map.put(key, value);
}
- public StateServicesSet get(Integer key) {
+ public StateServicesSet get(Long key) {
return map.get(key);
}
- public NavigableSet<Integer> getDescendingKeySet() {
+ public NavigableSet<Long> getDescendingKeySet() {
return map.descendingKeySet();
}
}