You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/10/28 19:12:54 UTC
svn commit: r1711088 [1/4] - in /uima/sandbox/uima-ducc/trunk:
src/main/admin/ src/main/assembly/ src/main/resources/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/per...
Author: challngr
Date: Wed Oct 28 18:12:53 2015
New Revision: 1711088
URL: http://svn.apache.org/viewvc?rev=1711088&view=rev
Log:
UIMA-4577 Switch from OrientDb to Cassandra.
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ReadCkpt.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java
uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Wed Oct 28 18:12:53 2015
@@ -115,6 +115,9 @@ class Ducc(DuccUtil):
def add_to_classpath(self, lib):
os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ":" + lib
+ def prepend_classpath(self, lib):
+ os.environ['CLASSPATH'] = lib + ":" + os.environ['CLASSPATH']
+
def run_component(self, component, or_parms, numagents, rmoverride, background, nodup, localdate):
if ( component == 'all' ):
@@ -181,7 +184,7 @@ class Ducc(DuccUtil):
jvm_opts.append("-Dducc.rm.override.dram=" + rmoverride)
if ( self.rm_jvm_args != None ):
jvm_opts.append(self.rm_jvm_args)
- self.add_to_classpath(ducc_home + '/lib/orientdb/*')
+ self.prepend_classpath(ducc_home + '/lib/cassandra/*')
if ( c == 'ws' ):
#see if the ws jsp compilation directory is specified
@@ -200,7 +203,7 @@ class Ducc(DuccUtil):
self.add_to_classpath(ducc_home + '/lib/http-client/*')
self.add_to_classpath(ducc_home + '/webserver/lib/*')
self.add_to_classpath(ducc_home + '/webserver/lib/jsp/*')
- self.add_to_classpath(ducc_home + '/lib/orientdb/*')
+ self.prepend_classpath(ducc_home + '/lib/cassandra/*')
if ( c == 'orchestrator' ):
if ( or_parms != None ):
@@ -209,7 +212,7 @@ class Ducc(DuccUtil):
jvm_opts.append(self.or_jvm_args)
self.add_to_classpath(ducc_home + '/lib/http-client/*')
self.add_to_classpath(ducc_home + '/webserver/lib/*')
- self.add_to_classpath(ducc_home + '/lib/orientdb/*')
+ self.prepend_classpath(ducc_home + '/lib/cassandra/*')
if ( c == 'pm' ):
if ( self.pm_jvm_args != None ):
@@ -221,7 +224,7 @@ class Ducc(DuccUtil):
self.add_to_classpath(ducc_home + '/apache-uima/apache-activemq/lib/optional/*')
self.add_to_classpath(ducc_home + '/lib/http-client/*')
self.add_to_classpath(ducc_home + '/webserver/lib/*')
- self.add_to_classpath(ducc_home + '/lib/orientdb/*')
+ self.prepend_classpath(ducc_home + '/lib/cassandra/*')
if (component != 'agent'):
service = 'org.apache.uima.ducc.common.main.DuccService'
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Oct 28 18:12:53 2015
@@ -239,6 +239,7 @@ class DuccUtil(DuccBase):
def db_start(self):
+ return True
# bypass all of this for the initial delivery
if ( self.db_parms == self.db_disabled ):
print ' (Bypass database start because ducc.database.host =', self.db_disabled + ')'
@@ -288,7 +289,7 @@ class DuccUtil(DuccBase):
return False
def db_init(self):
-
+ return True
# bypass all of this for the initial delivery
if ( self.db_parms == self.db_disabled ):
return True
@@ -304,6 +305,7 @@ class DuccUtil(DuccBase):
self.spawn(cmd)
def db_stop(self):
+ return True
# bypass all of this for the initial delivery
if ( self.db_parms == self.db_disabled ):
return True
Modified: uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml Wed Oct 28 18:12:53 2015
@@ -130,80 +130,29 @@ under the License.
<directoryMode>755</directoryMode>
</dependencySet>
- <!-- orientdb -->
- <dependencySet>
- <useStrictFiltering>true</useStrictFiltering>
- <includes>
- <include>com.tinkerpop.gremlin:gremlin-groovy</include>
- </includes>
- <outputDirectory>lib/orientdb</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <fileMode>644</fileMode>
- <directoryMode>755</directoryMode>
- </dependencySet>
-
- <dependencySet>
- <useStrictFiltering>true</useStrictFiltering>
- <includes>
- <include>com.tinkerpop.blueprints:blueprints-core</include>
- </includes>
- <outputDirectory>lib/orientdb</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <fileMode>644</fileMode>
- <directoryMode>755</directoryMode>
- </dependencySet>
<dependencySet>
<useStrictFiltering>true</useStrictFiltering>
<includes>
- <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include>
+ <include>com.datastax.cassandra:cassandra-driver-core</include>
+ <include>com.datastax.cassandra:cassandra-driver-dse</include>
+ <include>com.datastax.cassandra:cassandra-driver-mapping</include>
+ <include>io.netty:netty-buffer</include>
+ <include>io.netty:netty-codec</include>
+ <include>io.netty:netty-common</include>
+ <include>io.netty:netty-handler</include>
+ <include>io.netty:netty-transport</include>
+ <include>com.codahale.metrics:metrics-core</include>
+ <include>org.xerial.snappy:snappy-java</include>
+ <include>net.jpountz.lz4:lz4</include>
+ <include>org.hdrhistogram:HdrHistogram</include>
</includes>
- <outputDirectory>lib/orientdb</outputDirectory>
+ <outputDirectory>lib/cassandra</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<fileMode>644</fileMode>
<directoryMode>755</directoryMode>
</dependencySet>
- <dependencySet>
- <useStrictFiltering>true</useStrictFiltering>
- <includes>
- <include>com.orientechnologies:orientdb-graphdb</include>
- <include>com.orientechnologies:orientdb-core</include>
- <include>com.orientechnologies:orientdb-client</include>
- <include>com.orientechnologies:orientdb-server</include>
- <include>com.orientechnologies:orientdb-tools</include>
- <include>com.orientechnologies:orientdb-enterprise</include>
- </includes>
- <outputDirectory>lib/orientdb</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <fileMode>644</fileMode>
- <directoryMode>755</directoryMode>
- </dependencySet>
-
- <dependencySet>
- <useStrictFiltering>true</useStrictFiltering>
- <includes>
- <include>com.tinkerpop.gremlin:gremlin-groovy</include>
- </includes>
- <outputDirectory>lib/orientdb</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <fileMode>644</fileMode>
- <directoryMode>755</directoryMode>
- </dependencySet>
-
- <!-- dependencySet>
- <useStrictFiltering>true</useStrictFiltering>
- <includes>
- <include>com.orientechnologies:orientdb-lucene</include>
- </includes>
- <outputDirectory>database/plugins</outputDirectory>
- <useProjectArtifact>false</useProjectArtifact>
- <fileMode>644</fileMode>
- <directoryMode>755</directoryMode>
- </dependencySet -->
-
- <!-- end orientdb -->
-
<!-- xmlbeans is in UIMA-AS 2.6.0 package -->
<dependencySet>
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Wed Oct 28 18:12:53 2015
@@ -510,7 +510,7 @@ ducc.process.thread.sleep.time=3000
ducc.flow-controller.specifier=org.apache.uima.ducc.FlowController
-# database things -----------------------------------------------------------------------------------
+# database
# Directory where the databases and associated stuff is kept, relative to DUCC_HOME. This
# is ORIENTDB_HOME
ducc.database.location = database
@@ -520,16 +520,8 @@ ducc.database.config.file = resource
ducc.database.host = --disabled--
-# The binary url not HTTP/REST
-ducc.database.url = remote:${ducc.database.host}:2424
-# The rest interface
-ducc.database.rest.url = ${ducc.database.host}:2480
-
# Name of the database with active state. Svc registry, etc.
-ducc.state.database.url = ${ducc.database.url}/DuccState
-
-# Name of the database with history
-ducc.history.database.url = ${ducc.database.url}/DuccHistory
+ducc.state.database.url = ${ducc.head}
# This class implementes the persistence interface for the services registry
#ducc.service.persistence.impl = org.apache.uima.ducc.database.StateServicesDb
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java?rev=1711088&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java Wed Oct 28 18:12:53 2015
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
+package org.apache.uima.ducc.common.persistence;
+
+/**
+ * All properties sent to the DB must implement this interface.
+ */
+public interface IDbProperty
+{
+ String pname(); // This is the name of the property. Any java.util.Property key .toString() is valid.
+ Type type(); // One of the types below. DB impl translates to db equialent. Try to keep these
+ // an easy analogy of java names. DB does not convert so the type of the incoming
+ // propety has to match.
+ boolean isPrimaryKey(); // Primary key in DB. Many columns can have this which results in a compount key. Order
+ // is important - first is cluster key, and following are the others.
+ boolean isPrivate(); // Used only in the DB, do not pass back to the application in Properties. This goes into the db and
+ // is part of the schema.
+ boolean isMeta(); // A handy constant in the enum that isn't part of a schema. Helps in the generation
+ // of the schema. Doesn't go into the DB.
+ String columnName(); // Because we want to persist java.util.Properties, and these things
+ // allow a greater range of characters in the keys ('.', '-' and others)
+ // than does SQL, we define a translation from the "properties" key to
+ // legal SQL syntactic names. DB does not translate, user must provide
+ // a suitable translation.
+
+ // If we update this we may have to update db methods that use it
+ public enum Type {
+ String, // Java String
+ Blob, // Java serialized object or other binary
+ Boolean, // Java boolean
+ Integer,
+ Long,
+ Double,
+ UUID,
+ };
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java Wed Oct 28 18:12:53 2015
@@ -22,6 +22,7 @@ package org.apache.uima.ducc.common.pers
import java.util.Map;
import java.util.Properties;
+import org.apache.uima.ducc.common.persistence.IDbProperty;
import org.apache.uima.ducc.common.utils.DuccLogger;
public interface IRmPersistence
@@ -50,7 +51,7 @@ public interface IRmPersistence
* @param id This is the primary key, the machine name;
* @param properties These are the props, must be presented in the form of (String, Object) ...
*/
- public void setProperties(Object dbid, String id, Object... properties) throws Exception;
+ public void setProperties(String id, Object... properties) throws Exception;
/**
* Set a property on an object. If the property cannot be set the action
@@ -64,7 +65,7 @@ public interface IRmPersistence
* throw will originate in the DB because of some DB issue. An
* exception causes the action to be rolled back.
*/
- public void setProperty(Object dbid, String id, RmPropName key, Object value) throws Exception;
+ public void setProperty(String id, RmProperty key, Object value) throws Exception;
/**
@@ -82,7 +83,7 @@ public interface IRmPersistence
*
* @return The db id of the created machine.
*/
- public Object createMachine(String id, Properties props) throws Exception;
+ public void createMachine(String id, Map<RmProperty, Object> props) throws Exception;
/**
* Fetch a machine by its id.
@@ -110,58 +111,74 @@ public interface IRmPersistence
*/
public Map<String, Properties> getAllMachines() throws Exception;
- public String toGson(Object o);
- public interface RmProps
- {
- String pname();
- }
-
- enum RmPropName
- implements RmProps
+ enum RmProperty
+ implements IDbProperty
{
+ TABLE_NAME {
+ public String pname() { return "rmnodes"; }
+ public Type type() { return Type.String; }
+ public boolean isPrivate() { return true;}
+ public boolean isMeta() { return true;}
+ },
Name {
public String pname() { return "name"; }
+ public Type type() { return Type.String; }
+ public boolean isPrimaryKey() { return true;}
+
},
Responsive{
public String pname() { return "responsive"; }
+ public Type type() { return Type.Boolean; }
},
Online{
public String pname() { return "online"; }
- },
- HeartBeats {
- public String pname() { return "heartbeats"; }
+ public Type type() { return Type.Boolean; }
},
Ip {
public String pname() { return "ip"; }
+ public Type type() { return Type.String; }
},
Nodepool {
public String pname() { return "nodepool"; }
+ public Type type() { return Type.String; }
},
Quantum {
public String pname() { return "quantum"; }
+ public Type type() { return Type.Integer; }
},
Memory {
public String pname() { return "memory"; }
+ public Type type() { return Type.Integer; }
},
ShareOrder {
public String pname() { return "share_order"; }
+ public Type type() { return Type.Integer; }
},
Shares{
public String pname() { return "shares"; }
+ public Type type() { return Type.Integer; }
},
Blacklisted {
public String pname() { return "blacklisted"; }
+ public Type type() { return Type.Boolean; }
},
Heartbeats {
public String pname() { return "heartbeats"; }
+ public Type type() { return Type.Integer; }
},
SharesLeft {
public String pname() { return "shares_left"; }
+ public Type type() { return Type.Integer; }
},
Assignments {
public String pname() { return "assignments"; }
+ public Type type() { return Type.Integer; }
},
;
+ public boolean isPrimaryKey() { return false; }
+ public boolean isPrivate() { return false; }
+ public boolean isMeta() { return false; }
+ public String columnName() { return pname(); }
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java Wed Oct 28 18:12:53 2015
@@ -33,11 +33,9 @@ public class NullRmStatePersistence impl
public void init(DuccLogger logger) throws Exception { }
public void clear() {}
- public void setProperty(Object dbid, String id, RmPropName key, Object value) { }
- public void setProperties(Object dbid, String id, Object... props) {}
- public Object createMachine(String id, Properties props) { return new Integer(1);}
+ public void setProperty(String id, RmProperty key, Object value) { }
+ public void setProperties(String id, Object... props) {}
+ public void createMachine(String id, Map<RmProperty, Object> props) { }
public Properties getMachine(String id) { return null; }
public Map<String, Properties> getAllMachines() { return new HashMap<String, Properties>(); }
- public String toGson(Object o) { return ""; }
-
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java Wed Oct 28 18:12:53 2015
@@ -59,7 +59,7 @@ public class RmPersistenceFactory
ret = (IRmPersistence) iss.newInstance();
ret.init(logger);
} catch ( Throwable t ) {
- logger.error(methodName, null, "Cannot instantiate RM persistence class", clname, ":", t);
+ logger.error(methodName, null, "Cannot instantiate RM persistence class", clname, ":", t, "Using NullRmStatePersistance as default.");
ret = new NullRmStatePersistence();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java Wed Oct 28 18:12:53 2015
@@ -18,10 +18,10 @@
*/
package org.apache.uima.ducc.common.persistence.services;
-import java.util.List;
import java.util.Properties;
import org.apache.uima.ducc.common.IDuccEnv;
+import org.apache.uima.ducc.common.persistence.IDbProperty;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -31,104 +31,231 @@ public interface IStateServices {
public static String svc_reg_dir = IDuccEnv.DUCC_STATE_SVCREG_DIR;
public static String svc_hist_dir = IDuccEnv.DUCC_HISTORY_SVCREG_DIR;
-
- public static final String archive_key = "is_archived";
- public static final String archive_flag = "true";
-
public static final String svc = "svc";
public static final String meta = "meta";
- public enum SvcProps {
- endpoint { public String pname() { return "endpoint"; } },
- instances { public String pname() { return "instances"; } },
- autostart { public String pname() { return "autostart"; } },
- reference { public String pname() { return "reference"; } },
- enabled { public String pname() { return "enabled"; } },
+ public static final String UIMA_AS = "UIMA-AS";
+ public static final String CUSTOM = "CUSTOM";
+
+ public static final String sequenceKey = "service.seqno";
- disable_reason { public String pname() { return "disable_reason"; } },
+ // IMPORTANT IMPORTANT IMPORANT
+ // For DB: we must distinguish between properties for the registration itself, and
+ // metaprops.
+ // IMPORTANT IMPORTANT IMPORANT
+
+ // All this will be strings I guess, because they're all entered by humans
+ // and all the code is already debugged to deal with conversion.
+ public enum SvcRegProps
+ implements IDbProperty
+ {
+ TABLE_NAME {
+ public String pname() { return "smreg"; }
+ public Type type() { return Type.String; }
+ public boolean isPrivate() { return true; }
+ public boolean isMeta() { return true; }
+ },
+
+ numeric_id {
+ public String pname() { return "numeric_id"; }
+ public boolean isPrimaryKey() { return true; }
+ public boolean isPrivate() { return true; }
+
+ },
+
+ uuid {
+ public String pname() { return "uuid"; }
+ public boolean isPrivate() { return true; }
+ },
+
+ is_archived {
+ public String pname() { return "is_archived"; }
+ public Type type() { return Type.Boolean; }
+ public boolean isPrivate() { return true; }
+ },
+
+ description { public String pname() { return "description"; } },
+ administrators { public String pname() { return "administrators"; } },
+ scheduling_class { public String pname() { return "scheduling_class"; } },
+ log_directory { public String pname() { return "log_directory"; } },
+ working_directory { public String pname() { return "working_directory"; } },
+ jvm { public String pname() { return "jvm"; } },
+ process_jvm_args { public String pname() { return "process_jvm_args"; } },
+ classpath { public String pname() { return "classpath"; } },
+ environment { public String pname() { return "environment"; } },
+
+ process_memory_size { public String pname() { return "process_memory_size"; } },
+ process_dd { public String pname() { return "process_descriptor_DD"; } },
+ process_debug { public String pname() { return "process_debug"; } },
+ process_executable { public String pname() { return "process_executable"; } },
+ process_executable_args { public String pname() { return "process_executable_args"; } },
+ process_initialization_time_max { public String pname() { return "process_initialization_time_max"; } },
+
+ service_dependency { public String pname() { return "service_dependency"; } },
+ service_request_endpoint { public String pname() { return "service_request_endpoint"; } },
+ service_linger { public String pname() { return "service_linger"; } },
+
+ service_ping_arguments { public String pname() { return "service_ping_arguments"; } },
+ service_ping_class { public String pname() { return "service_ping_class"; } },
+ service_ping_classpath { public String pname() { return "service_ping_classpath"; } },
+ service_ping_jvm_args { public String pname() { return "service_ping_jvm_args"; } },
+ service_ping_dolog { public String pname() { return "service_ping_dolog"; } },
+ service_ping_timeout { public String pname() { return "service_ping_timeout"; } },
+
+ instance_failures_window { public String pname() { return "instance_failures_window"; } },
+ instance_failures_limit { public String pname() { return "instance_failures_limit"; } },
+ instance_init_failures_limit { public String pname() { return "instance_init_failures_limit"; } },
+
+ ;
+ public Type type() { return Type.String; }
+ public boolean isPrimaryKey() { return false; }
+ public boolean isPrivate() { return false; }
+ public boolean isMeta() { return false; }
+ public String columnName() { return pname(); }
+ };
+
+ // These will be strings as well because there's code all over assuming they're strings.
+ public enum SvcMetaProps
+ implements IDbProperty
+ {
+ TABLE_NAME {
+ public String pname() { return "smmeta"; }
+ public Type type() { return Type.String; }
+ public boolean isPrivate() { return true; }
+ public boolean isMeta() { return true; }
+ },
+
+ numeric_id {
+ public String pname() { return "numeric_id"; }
+ public boolean isPrimaryKey() { return true; }
+ },
+
+ is_archived {
+ public String pname() { return "is_archived"; }
+ public Type type() { return Type.Boolean; }
+ public boolean isPrivate() { return true; }
+ },
- implementors { public String pname() { return "implementors"; } },
- numeric_id { public String pname() { return "numeric_id"; } },
uuid { public String pname() { return "uuid"; } },
- service_seqno { public String pname() { return "service.seqno"; } },
- ping_active { public String pname() { return "ping_active"; } },
- ping_only { public String pname() { return "ping_only"; } },
-
- service_alive { public String pname() { return "service_alive"; } },
- service_class { public String pname() { return "service_class"; } },
- service_dependency { public String pname() { return "service_dependency"; } },
- service_healthy { public String pname() { return "service_healthy"; } },
- service_state { public String pname() { return "service_state"; } },
- last_use { public String pname() { return "last_use"; } },
- last_use_readable { public String pname() { return "last_use_readable"; } },
- service_statistics { public String pname() { return "service_statistics"; } },
- service_type { public String pname() { return "service_type"; } },
- submit_error { public String pname() { return "submit_error"; } },
- user { public String pname() { return "user"; } },
- references { public String pname() { return "references"; } },
- stopped { public String pname() { return "stopped"; } },
-
- scheduling_class { public String pname() { return "scheduling_class"; } },
- process_memory_size { public String pname() { return "process_memory_size"; } },
- description { public String pname() { return "description"; } },
- log_directory { public String pname() { return "log_directory"; } },
- process_executable { public String pname() { return "process_executable"; } },
-
- last_ping { public String pname() { return "last_ping"; } },
- last_ping_readable { public String pname() { return "last_ping_readable"; } },
- last_runnable { public String pname() { return "last_runnable"; } },
- last_runnable_readable { public String pname() { return "last_runnable_readable"; } },
- work_instances { public String pname() { return "work_instances"; } },
- registration_date { public String pname() { return "registration_date"; } },
- registration_date_millis { public String pname() { return "registration_date_millis"; } },
+ reference { public String pname() { return "reference"; } }, // boolean, is this reference s
+ enabled { public String pname() { return "enabled"; } },
+ autostart { public String pname() { return "autostart"; } }, // here we specify whether it is autostarted
+ instances { public String pname() { return "instances"; } },
+ endpoint { public String pname() { return "endpoint"; } },
- instance_init_failures_limit { public String pname() { return "instance_init_failures_limit"; } },
+ disable_reason {
+ public String pname() { return "disable-reason"; }
+ public String columnName() { return "disable_reason"; }
+ },
+
+ implementors {
+ public String pname() { return "implementors"; }
+ },
+
+ ping_active {
+ public String pname() { return "ping-active"; }
+ public String columnName() { return "ping_active"; }
+ },
+ ping_only {
+ public String pname() { return "ping-only"; }
+ public String columnName() { return "ping_only"; }
+ },
+
+ service_alive {
+ public String pname() { return "service-alive"; }
+ public String columnName() { return "service_alive"; }
+ },
+ service_class {
+ public String pname() { return "service-class"; }
+ public String columnName() { return "service_class"; }
+ },
+ service_dependency {
+ public String pname() { return "service-dependency"; }
+ public String columnName() { return "service_dependency"; }
+ },
+ service_healthy {
+ public String pname() { return "service-healthy"; }
+ public String columnName() { return "service_healthy"; }
+ },
+ service_state {
+ public String pname() { return "service-state"; }
+ public String columnName() { return "service_state"; }
+ },
+ last_use {
+ public String pname() { return "last-use"; }
+ public String columnName() { return "last_use"; }
+ },
+ last_use_readable {
+ public String pname() { return "last-use-readable"; }
+ public String columnName() { return "last_use_readable"; }
+ },
+ service_statistics {
+ public String pname() { return "service-statistics"; }
+ public String columnName() { return "service_statistics"; }
+ },
+ service_type {
+ public String pname() { return "service-type"; }
+ public String columnName() { return "service_type"; }
+ },
+ submit_error {
+ public String pname() { return "submit-error"; }
+ public String columnName() { return "submit_error"; }
+ },
+ user {
+ public String pname() { return "user"; }
+ },
- UIMA_AS { public String pname() { return "UIMA-AS"; } },
- CUSTOM { public String pname() { return "CUSTOM"; } },
+ references { public String pname() { return "references"; } }, // things that reference me
+ stopped { public String pname() { return "stopped"; } },
+
+ last_ping {
+ public String pname() { return "last-ping"; }
+ public String columnName() { return "last_ping"; }
+ },
+ last_ping_readable {
+ public String pname() { return "last-ping-readable"; }
+ public String columnName() { return "last_ping_readable"; }
+ },
+ last_runnable {
+ public String pname() { return "last-runnable"; }
+ public String columnName() { return "last_runnable"; }
+ },
+ last_runnable_readable {
+ public String pname() { return "last-runnable-readable"; }
+ public String columnName() { return "last_runnable_readable"; }
+ },
+ work_instances {
+ public String pname() { return "work-instances"; }
+ public String columnName() { return "work_instances"; }
+ },
+ registration_date {
+ public String pname() { return "registration-date"; }
+ public String columnName() { return "registration_date"; }
+ },
+ registration_date_millis {
+ public String pname() { return "registration-date-millis"; }
+ public String columnName() { return "registration_date_millis"; }
+ },
;
- public abstract String pname();
- };
+ public Type type() { return Type.String; }
+ public boolean isPrimaryKey() { return false; }
+ public boolean isPrivate() { return false; }
+ public boolean isMeta() { return false; }
+ public String columnName() { return pname(); }
- //public static final String endpoint { public String pname() { return "endpoint"; } },
- //public static final String instances = "instances";
- //public static final String autostart = "autostart";
-
- //public static final String reference = "reference";
- //public static final String enabled = "enabled";
- //public static final String disable_reason = "disable-reason";
- //public static final String implementors = "implementors";
- //public static final String numeric_id = "numeric_id";
- //public static final String ping_active = "ping-active";
- //public static final String ping_only = "ping-only";
- //public static final String service_alive = "service-alive";
- //public static final String service_class = "service-class";
- //public static final String service_dependency = "service_dependency";
- //public static final String service_healthy = "service-healthy";
- //public static final String service_state = "service-state";
- //public static final String last_use = "last-use";
- //public static final String service_statistics = "service-statistics";
- //public static final String service_type = "service-type";
- //public static final String submit_error = "submit-error";
- //public static final String user = "user";
-
- //public static final String scheduling_class = "scheduling_class";
- //public static final String process_memory_size = "process_memory_size";
- //public static final String description = "description";
- //public static final String log_directory = "log_directory";
- //public static final String process_executable = "process_executable";
-
- //public static final String service_type_UIMA_AS = "UIMA-AS";
- //public static final String service_type_CUSTOM = "CUSTOM";
+
+ };
//
- // IMPORTANT IMPORTANT IMPORTANT
- // As of now, 2015/08/17, there is no support for service registration history. There never has been in
- // this class and it's derivatives. Until there is a use-case and demand it's not clear what the interfaces
- // would be like.
+ // IMPORTANT IMPORTANT IMPORTANT
+ //
+ // As of now, 2015/08/17, there is no support for service registration history in this class.
+ // There never has been direct DUCC support for this other than keeping the old registrations
+ // around. Until there is a use-case and demand it's not clear what the interfaces would be
+ // like.
//
// We may have to prune, or look at specific history files, but that can be done with direct database
// operations. If we need more, well add it later.
@@ -137,15 +264,16 @@ public interface IStateServices {
// 'history' state.
// IMPORTANT IMPORTANT IMPORTANT
- public List<Long> getSvcList() throws Exception; // list of registered services
- public List<Long> getMetaList() throws Exception; // not used ?
+ // Never used except in test. Removing them pending complaints
+ //public List<Long> getSvcList() throws Exception; // list of registered services
+ //public List<Long> getMetaList() throws Exception; // not used ?
public StateServicesDirectory getStateServicesDirectory() throws Exception; // all the registy in one blow
public boolean storeProperties (DuccId serviceId, Properties svc, Properties meta) throws Exception; // save svc and meta in a transaction
- public boolean updateJobProperties (Object dbid, DuccId serviceId, Properties props) throws Exception; // update just job props
- public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props) throws Exception; // update just metaprops
- public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception;
+ public boolean updateJobProperties (DuccId serviceId, Properties props) throws Exception; // update just job props
+ public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception; // update just metaprops
+ public boolean moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception;
public void shutdown() throws Exception;
public boolean init(DuccLogger logger) throws Exception;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java Wed Oct 28 18:12:53 2015
@@ -19,7 +19,6 @@
package org.apache.uima.ducc.common.persistence.services;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Properties;
import org.apache.uima.ducc.common.utils.DuccLogger;
@@ -33,33 +32,19 @@ public class NullStateServices implement
}
public boolean init(DuccLogger logger) throws Exception { return true; }
- public ArrayList<Long> getSvcList()
- throws Exception
- {
- return new ArrayList<Long>();
- }
-
-
- public ArrayList<Long> getMetaList()
- throws Exception
- {
- return new ArrayList<Long>();
- }
-
-
+
public StateServicesDirectory getStateServicesDirectory()
throws IOException
{
return new StateServicesDirectory();
}
- public boolean storeProperties(DuccId serviceId, Properties svc, Properties meta) throws Exception {return false;}
- public boolean updateProperties(Object dbid, DuccId serviceId, String type, Properties props) throws Exception {return false;}
+ public boolean storeProperties(DuccId serviceId, Properties svc, Properties meta) throws Exception {return true;}
public void deleteProperties(DuccId serviceId) throws Exception {}
public void shutdown() throws Exception {}
public void moveToHHistory() throws Exception {}
- public boolean updateJobProperties(Object dbid, DuccId serviceId, Properties props) throws Exception {return false;}
- public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props) throws Exception {return false;}
- public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception {}
+ public boolean updateJobProperties(DuccId serviceId, Properties props) throws Exception {return true;}
+ public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception {return true;}
+ public boolean moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception {return true;}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java Wed Oct 28 18:12:53 2015
@@ -89,14 +89,14 @@ public class StateServices implements IS
}
- public ArrayList<Long> getSvcList() {
+ private ArrayList<Long> getSvcList() {
return getList(IStateServices.svc);
}
- public ArrayList<Long> getMetaList() {
- return getList(IStateServices.meta);
- }
+// private ArrayList<Long> getMetaList() {
+// return getList(IStateServices.meta);
+// }
private DuccProperties getProperties(String name) {
String location = "getProperties";
@@ -237,21 +237,21 @@ public class StateServices implements IS
return ok;
}
- private boolean updateProperties(Object dbid, DuccId serviceId, Properties props, String type)
+ private boolean updateProperties(DuccId serviceId, Properties props, String type)
{
File f = new File(mkfilename(serviceId, type));
File tmpf = new File(f.toString() + ".tmp");
return saveProperties(serviceId, props, f, tmpf, type);
}
- public boolean updateJobProperties(Object dbid, DuccId serviceId, Properties props)
+ public boolean updateJobProperties(DuccId serviceId, Properties props)
{
- return updateProperties(dbid, serviceId, props, svc);
+ return updateProperties(serviceId, props, svc);
}
- public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props)
+ public boolean updateMetaProperties(DuccId serviceId, Properties props)
{
- return updateProperties(dbid, serviceId, props, meta);
+ return updateProperties(serviceId, props, meta);
}
public void deleteProperties(long serviceId)
@@ -272,11 +272,12 @@ public class StateServices implements IS
}
- public void moveToHistory(DuccId id, Properties svc, Properties meta)
+ public boolean moveToHistory(DuccId id, Properties svc, Properties meta)
throws Exception
{
String methodName = "moveToHistory";
+ boolean ret = true;
// Save a copy in history, and then delete the original
File mfh = new File(svc_hist_dir + id + ".meta");
try {
@@ -300,12 +301,15 @@ public class StateServices implements IS
fos.close();
} catch (Exception e) {
logger.warn(methodName, null, id + ":Unable to save history to \"" + pfh.toString(), ": ", e.toString() + "\"");
+ ret = false;
}
String props_filename = svc_reg_dir + id + ".svc";
File pf = new File(props_filename);
pf.delete();
+ return ret;
}
public void shutdown() {}
}
+
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java Wed Oct 28 18:12:53 2015
@@ -18,6 +18,7 @@
*/
package org.apache.uima.ducc.common.persistence.services;
+import java.util.Map;
import java.util.NavigableSet;
import java.util.TreeMap;
@@ -36,4 +37,8 @@ public class StateServicesDirectory {
public NavigableSet<Long> getDescendingKeySet() {
return map.descendingKeySet();
}
+
+ public Map<Long, StateServicesSet> getMap() {
+ return map;
+ }
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java Wed Oct 28 18:12:53 2015
@@ -19,7 +19,6 @@
package org.apache.uima.ducc.common.test.cmd;
import java.util.Iterator;
-import java.util.List;
import java.util.Map.Entry;
import java.util.NavigableSet;
import java.util.Properties;
@@ -45,14 +44,16 @@ public class StateServicesTest
return;
}
IStateServices ss = StateServicesFactory.getInstance(StateServicesTest.class.getName(), "TEST");
- List<Long> svcList = ss.getSvcList();
- for(Long fname : svcList) {
- System.out.println(fname);
- }
- List<Long> metaList = ss.getMetaList();
- for(Long fname : metaList) {
- System.out.println(fname);
- }
+
+ // These two routines are never used except in this test! Removing them pending complaints.
+// List<Long> svcList = ss.getSvcList();
+// for(Long fname : svcList) {
+// System.out.println(fname);
+// }
+// List<Long> metaList = ss.getMetaList();
+// for(Long fname : metaList) {
+// System.out.println(fname);
+// }
StateServicesDirectory ssd = ss.getStateServicesDirectory();
NavigableSet<Long> keySet = ssd.getDescendingKeySet();
Iterator<Long> iterator = keySet.iterator();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml Wed Oct 28 18:12:53 2015
@@ -63,36 +63,39 @@
<artifactId>uima-ducc-transport</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>${cassandra.driver.version}</version>
+ </dependency>
<dependency>
- <groupId>com.orientechnologies</groupId>
- <artifactId>orientdb-graphdb</artifactId>
- <version>${orientdb.version}</version>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-dse</artifactId>
+ <version>${cassandra.driver.version}</version>
</dependency>
-
+
<dependency>
- <groupId>com.orientechnologies</groupId>
- <artifactId>orientdb-lucene</artifactId>
- <version>${orientdb.version}</version>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-mapping</artifactId>
+ <version>${cassandra.driver.version}</version>
</dependency>
-
+
<dependency>
- <groupId>com.tinkerpop.blueprints</groupId>
- <artifactId>blueprints-core</artifactId>
- <version>2.6.0</version>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
</dependency>
<dependency>
- <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
- <artifactId>concurrentlinkedhashmap-lru</artifactId>
- <version>1.4.2</version>
- </dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ </dependency>
<dependency>
- <groupId>com.tinkerpop.gremlin</groupId>
- <artifactId>gremlin-groovy</artifactId>
- <version>2.6.0</version>
+ <groupId>org.hdrhistogram</groupId>
+ <artifactId>HdrHistogram</artifactId>
</dependency>
+
</dependencies>
</project>
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java Wed Oct 28 18:12:53 2015
@@ -19,18 +19,15 @@
package org.apache.uima.ducc.database;
+import java.util.List;
+
import org.apache.uima.ducc.common.utils.DuccLogger;
-import org.apache.uima.ducc.database.DbConstants.DbEdge;
-import org.apache.uima.ducc.database.DbConstants.DbVertex;
-import org.apache.uima.ducc.database.DbConstants.Index;
-
-import com.orientechnologies.orient.client.remote.OServerAdmin;
-import com.orientechnologies.orient.core.metadata.schema.OProperty;
-import com.orientechnologies.orient.core.sql.OCommandSQL;
-import com.tinkerpop.blueprints.impls.orient.OrientEdgeType;
-import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory;
-import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx;
-import com.tinkerpop.blueprints.impls.orient.OrientVertexType;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Metadata;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
public class DbCreate
{
@@ -38,8 +35,9 @@ public class DbCreate
String dburl;
String adminid = "root";
String adminpw = null;
- OServerAdmin admin;
- OrientGraphFactory factory;
+
+ private Cluster cluster;
+ private Session session;
public DbCreate(String dburl)
{
@@ -60,6 +58,31 @@ public class DbCreate
this.adminpw = adminpw;
}
+ public void connect()
+ {
+ String methodName = "connect";
+ cluster = Cluster.builder()
+ .addContactPoint(dburl)
+ .build();
+
+ Metadata metadata = cluster.getMetadata();
+ doLog(methodName, "Connected to cluster: %s\n", metadata.getClusterName());
+
+ for ( Host host : metadata.getAllHosts() ) {
+ doLog(methodName, "Datatacenter: %s; Host: %s; Rack: %s\n", host.getDatacenter(), host.getAddress(), host.getRack());
+ }
+ session = cluster.connect();
+ }
+
+ public void close() {
+ cluster.close();
+ }
+
+ public Session getSession()
+ {
+ return this.session;
+ }
+
public void doLog(String methodName, Object ... msg)
{
if ( logger == null ) {
@@ -72,8 +95,7 @@ public class DbCreate
} else {
buf.append(o.toString());
}
- }
-
+ }
System.out.println(buf);
} else {
logger.info(methodName, null, msg);
@@ -82,162 +104,74 @@ public class DbCreate
}
- void createEdgeType(OrientGraphNoTx g, DbEdge id)
+ String mkTableCreate(String tableName, String[] fields)
{
- String methodName = "createEdgeType";
- String s = id.pname();
- OrientEdgeType e = g.getEdgeType(s);
- if ( e == null ) {
- doLog(methodName, "Create edge", s);
- g.createEdgeType(s);
+ int max = fields.length - 1;
+ int current = 0;
+ StringBuffer buf = new StringBuffer("CREATE TABLE IF NOT EXISTS ");
+ buf.append(tableName);
+ buf.append(" (");
+ for (String s : fields) {
+ buf.append(s);
+ if ( current++ < max) buf.append(", ");
}
+ buf.append(") WITH CLUSTERING ORDER BY (ducc_dbid desc)");
+ return buf.toString();
}
-// void createVertexType(OrientGraphNoTx g, DbVertex id)
-// {
-// String methodName = "createVertexType";
-// String s = id.pname();
-// OrientVertexType e = g.getVertexType(s);
-// if ( e == null ) {
-// doLog(methodName, "Create vertex " + s);
-// e = g.createVertexType(s, DuccVertexBase.VBase.pname());
-// }
-// }
-
- void createVertices(OrientGraphNoTx g)
- {
- String methodName = "createVertices";
-
- for (DbVertex v : DbVertex.values()) {
- String s = v.pname();
- OrientVertexType vt = g.getVertexType(s);
-
-
- if ( vt == null ) {
- if ( v.parent() == null ) {
- vt = g.createVertexType(s);
- doLog(methodName, "Create vertex " + s);
- } else {
- vt = g.createVertexType(s, v.parent().pname());
- doLog(methodName, "Create vertex ", s, "subclass from", v.parent().pname());
- }
-
- Index[] ndx = v.indices();
- if ( ndx != null ) {
- for ( Index i : ndx ) {
- OProperty p = vt.createProperty(i.propname(), i.type());
- p.setMandatory(true);
- doLog(methodName, "Create property", i.propname(), "on", s);
- }
- }
-
- // String sql = "create index " + i.pname() + " on " + s + "(" + i.propname() + " by key) notunique";
- // g.command(new OCommandSQL(sql)).execute();
- // doLog(methodName, "(sql)Created index", i.pname(), "on class", s, "for", i.propname());
- // //g.createKeyIndex(i.propname(), Vertex.class, new Parameter("type", "NOTUNIQUE_HASH_INDEX"));
- // doLog(methodName, "(java)Created index on class Vertex for", i.propname());
- // }
- // }
-
- }
- }
-
- if ( true ) return;
- String sql = "create index D.VBASE_0 on VWork(ducc_dbid, ducc_dbcat) NOTUNIQUE";
- g.command(new OCommandSQL(sql)).execute();
-
- String sql1 = "create index D.VBASE_ID on VWork(ducc_dbid) NOTUNIQUE";
- g.command(new OCommandSQL(sql1)).execute();
-
- String sql2 = "create index D.VBASE_CAT on VWork(ducc_dbcat) NOTUNIQUE";
- g.command(new OCommandSQL(sql2)).execute();
-
- String sql3 = "create index D.NODE on VRmNode(ducc_dbnode) NOTUNIQUE";
- g.command(new OCommandSQL(sql3)).execute();
-
- }
-
- void createEdges(OrientGraphNoTx g)
- {
- String methodName = "createEdges";
- for (DbEdge e : DbEdge.values()) {
- String s = e.pname();
- OrientEdgeType et = g.getEdgeType(s);
- if ( et == null ) {
- if ( e.parent() == null ) {
- et = g.createEdgeType(s);
- doLog(methodName, "Create edge " + s);
- } else {
- et = g.createEdgeType(s, e.parent().pname());
- doLog(methodName, "Create edge ", s, "subclass from", e.parent().pname());
- }
- }
- }
- }
-
void createSchema()
{
- OrientGraphNoTx g = factory.getNoTx();
+ String methodName = "createSchema";
+
+ // A 'keyspace' is what we usually think of as a database.
+ session.execute("CREATE KEYSPACE IF NOT EXISTS ducc WITH replication = {'class':'SimpleStrategy', 'replication_factor':1};");
try {
- createVertices(g);
- createEdges(g);
- } finally {
- g.shutdown();
- }
- }
+ List<SimpleStatement>rmSchema = RmStatePersistence.mkSchema();
+ for ( SimpleStatement s : rmSchema ) {
+ doLog(methodName, "EXECUTE STATEMENT:", s.toString());
+ session.execute(s);
+ }
- // void createSchemax()
- // {
- // String methodName = "createSchema";
- // OrientGraphNoTx g = factory.getNoTx();
-
- // String base = DuccVertexBase.VBase.pname();
- // OrientVertexType e = g.getVertexType(base);
- // if ( e == null ) {
- // doLog(methodName, "Create base vertex class " + base);
- // e = g.createVertexType(base);
- // OProperty p = e.createProperty(DbConstants.DUCCID, OType.LONG);
- // p.setMandatory(true);
- // OProperty p2 = e.createProperty(DbConstants.DUCC_DBCAT, OType.STRING);
- // p2.setMandatory(true);
-
- // String sql = "create index i_ducc_dbid on " + base + "(" + DbConstants.DUCCID + ") notunique";
- // g.command(new OCommandSQL(sql)).execute();
- // doLog(methodName, "(sql)Created index i_ducc_dbid on class " + base + " for " + DbConstants.DUCCID);
-
- // sql = "create index i_ducc_dbcat on " + base + "(" + DbConstants.DUCC_DBCAT + ") notunique";
- // g.command(new OCommandSQL(sql)).execute();
- // doLog(methodName, "(sql)Created index i_ducc_dbcat on class " + base + " for " + DbConstants.DUCC_DBCAT);
+ //
+ // String[] rmSchema = RmStatePersistence.mkSchemaItems();
+ // String cql = DbUtil.mkTableCreate("ducc.rmnodes", rmSchema);
+ // doLog(methodName, "CQL:", cql);
+ // session.execute(cql);
+
+ List<SimpleStatement>smSchema = StateServicesDb.mkSchema();
+ for ( SimpleStatement s : smSchema ) {
+ doLog(methodName, "EXECUTE STATEMENT:", s.toString());
+ session.execute(s);
+ }
- // }
-
- // for ( DbVertex o : DbVertex.values() ) {
- // createVertexType(g, o);
- // }
- // for ( DbEdge o : DbEdge.values() ) {
- // createEdgeType(g, o);
- // }
-
- // g.shutdown();
- // }
- boolean createPlocalDatabase()
- throws Exception
- {
- boolean ret = false;
- try {
- factory = new OrientGraphFactory(dburl, "admin", "admin");
- createSchema();
- ret = true;
+// String[] smSchemaReg = StateServicesDb.mkSchemaForReg();
+// cql = DbUtil.mkTableCreate("ducc.smreg", smSchemaReg);
+// doLog(methodName, "CQL:", cql);
+// session.execute(cql);
+// cql = "CREATE INDEX IF NOT EXISTS ON ducc.smreg(active)";
+// session.execute(cql);
+//
+// String[] smSchemaMeta = StateServicesDb.mkSchemaForMeta();
+// cql = DbUtil.mkTableCreate("ducc.smmeta", smSchemaMeta);
+// doLog(methodName, "CQL:", cql);
+// session.execute(cql);
+// cql = "CREATE INDEX IF NOT EXISTS ON ducc.smmeta(active)";
+// session.execute(cql);
+
+ List<SimpleStatement>orSchema = HistoryManagerDb.mkSchema();
+ for ( SimpleStatement s : orSchema ) {
+ doLog(methodName, "EXECUTE STATEMENT:", s.toString());
+ session.execute(s);
+ }
+
} catch ( Exception e ) {
- e.printStackTrace();
- } finally {
- factory.close();
+ doLog(methodName, "Cannot create schema:", e);
}
- return ret;
+
}
-
+
/**
* Create the database and initialize the schema. This is intended to be called only from Main at
* system startup, to insure all users of the db have a db when they start.
@@ -245,36 +179,8 @@ public class DbCreate
boolean createDatabase()
throws Exception
{
- String methodName = "createDatabase";
-
- if ( adminpw == null ) {
- adminpw = DbManager.dbPassword();
- }
+ //String methodName = "createDatabase";
- try {
- admin = new OServerAdmin(dburl);
- admin.connect(adminid, adminpw); // connect to the server
-
- if ( ! admin.existsDatabase("plocal") ) {
- doLog(methodName, "Database " + dburl + " does not exist, attempting to create it.");
- admin.createDatabase("graph", "plocal");
-
- if ( ! admin.existsDatabase() ) {
- doLog(methodName, "Cannot create database " + dburl);
- return false;
- }
- factory = new OrientGraphFactory(dburl);
- if ( factory == null ) {
- doLog(methodName, "Cannot create graph factory for " + dburl);
- return false;
- }
-
- createSchema();
- }
- } finally {
- if ( admin != null ) admin.close();
- if ( factory != null ) factory.close();
- }
return true;
}
@@ -284,17 +190,20 @@ public class DbCreate
System.out.println("Usage: DbCreate <database url>");
System.exit(1);
}
+
+ DbCreate client = null;
try {
- DbCreate dbc = new DbCreate(args[0]);
- if ( ! dbc.createDatabase() ) {
- System.out.println("Could not create database or schema for " + args[0]);
- System.exit(1);
- }
- } catch ( Exception e ) {
+ client = new DbCreate(args[0]);
+ client.connect();
+ client.createSchema();
+ } catch ( Throwable e ) {
System.out.println("Errors creating database");
e.printStackTrace();
System.exit(1);
+ } finally {
+ if ( client != null ) client.close();
}
+
System.exit(0);
}